00000800:00000200:0.1F:1767315028.201037:0:7009:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001254e72c] -> 12345-192.168.204.107@tcp (2) 00000800:00000200:3.1F:1767315028.206015:0:7000:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001254e72c] -> 12345-192.168.204.107@tcp (2) 00000800:00000200:3.1:1767315028.206026:0:7000:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.204.107@tcp 00000000eabca2f1 deleted 00000800:00000010:3.1:1767315028.206028:0:7000:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 00000000eabca2f1. 00000800:00000200:3.1:1767315028.206033:0:7000:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1767315028.206039:0:7000:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.107@tcp ip 192.168.204.107:1023 00000800:00000200:3.1:1767315028.206043:0:7000:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000f3e00e4a type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.1:1767315028.206463:0:6999:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1F:1767315028.207705:0:7002:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000001254e72c] -> 12345-192.168.204.107@tcp (4) 00000800:00000200:2.1:1767315028.207713:0:7002:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.204.107@tcp 0000000037aca84e deleted 00000800:00000010:2.1:1767315028.207715:0:7002:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at 0000000037aca84e. 00000800:00000200:2.1:1767315028.207719:0:7002:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected a ksnr_max_conns 1 00000800:00000200:3.1:1767315028.210547:0:6999:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected e ksnr_max_conns 1 00010000:00000010:1.1F:1767315031.499680:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000a75ae040. 00010000:00000010:1.1:1767315031.499731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000006cf34752. 00010000:00000010:2.1:1767315031.990503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000000f325957. 00010000:00000010:2.1:1767315031.990519:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 000000001aec8c4d. 00010000:00000010:2.1:1767315031.990523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000005afe94e1. 00010000:00000010:0.1:1767315031.992212:0:15498:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 0000000057b1f3f2. 00010000:00000010:0.1:1767315031.992217:0:15498:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000001a6fc8a6. 00010000:00000010:0.1:1767315032.194599:0:15508:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000061349ee0. 00010000:00000010:0.1:1767315032.194607:0:15508:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000b5719ac0. 00010000:00000010:2.1:1767315032.194789:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000e88dadd4. 00010000:00000010:2.1:1767315032.194795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000097d400d2. 00010000:00000010:1.1:1767315032.195788:0:15395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000008eadd25c. 00010000:00000010:1.1:1767315032.207853:0:788:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000a663a325. 00010000:00000010:1.1:1767315032.207860:0:788:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000068191b8f. 00010000:00000010:1.1:1767315032.207878:0:788:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 000000004a5b0bed. 00010000:00000010:1.1:1767315032.207883:0:788:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 000000001a6fc8a6. 00010000:00000010:3.1:1767315032.685475:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 0000000016262d3e. 00010000:00000010:3.1:1767315032.685489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000fbc9ae96. 00010000:00000010:1.1:1767315032.699454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 00000000e6ea0095. 00010000:00000010:1.1:1767315032.699472:0:0:0:(ldlm_resource.c:1330:__ldlm_resource_free()) slab-freed '(res)': 248 at 00000000f93a6d73. 00010000:00000010:1.1:1767315032.699476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at 0000000063900160. 00080000:00000010:0.1:1767315032.731483:0:0:0:(osd_handler.c:1617:osd_oxc_free()) kfreed 'oxe': 68 at 00000000d047002a. 00000100:00080000:0.0:1767318377.412757:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318377.412771:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318377.413118:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318377.413124:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318377.413129:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318377.413138:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3620 00000100:00080000:0.0:1767318377.413145:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318377.413153:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318377.413187:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56a00 x1853163982544512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318377.413225:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318377.413228:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318377.413230:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318377.413234:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3620 00000100:00080000:0.0:1767318377.413237:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318377.413241:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318377.413254:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54e00 x1853163982544768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318377.413263:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318377.413267:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318377.413302:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318377.413306:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3620 00000100:00080000:0.0:1767318377.413308:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318377.413313:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318377.413330:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54a80 x1853163982544896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318377.413364:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318377.413367:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318377.413370:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318377.413373:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3620 00000100:00080000:0.0:1767318377.413375:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318377.413380:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318377.413390:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56680 x1853163982545280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318377.415155:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318377.415164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318377.416237:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 179 00000100:00080000:1.0:1767318377.416323:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318377.416329:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318377.416332:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982544896:0@lo:38:kworker.0 00010000:00080000:1.0:1767318377.416401:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54000 x1853163982544640/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318393 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318377.416414:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982544640:0@lo:400:kworker.0 00010000:00080000:1.0:1767318377.416425:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54380 x1853163982544384/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318393 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318377.416433:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982544384:0@lo:400:kworker.0 00010000:00010000:1.0:1767318377.540610:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318378.564705:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318378.948945:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982545408:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318378.949049:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318378.949134:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318378.949141:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982545536:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318378.949171:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318380.432693:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318380.432746:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:1.0:1767318380.438856:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318380.613183:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318381.636631:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318382.532878:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982545792:0@lo:38:kworker.0 00000100:00100000:0.0:1767318382.532884:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982545664:0@lo:400:kworker.0 00000100:00100000:1.0:1767318382.540165:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318382.540213:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982546048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318382.540252:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318382.541887:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982546176:0@lo:38:kworker.0 00000100:00100000:1.0:1767318382.541932:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318382.541963:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982546560:0@lo:38:kworker.0 00000100:00100000:1.0:1767318382.541998:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318382.543872:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982546176 00000100:00100000:0.0:1767318382.543884:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982546176 00000100:00100000:0.0:1767318382.543929:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 180 00000100:00100000:0.0:1767318382.543937:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982546176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318382.543973:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318382.544023:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982546176:12345-0@lo:38:kworker.0 Request processed in 86us (2090us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318382.544030:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 180 00000100:00100000:0.0:1767318382.548502:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982546048 00000100:00100000:0.0:1767318382.548509:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982546048 00000100:00100000:0.0:1767318382.548561:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 619 00000100:00100000:0.0:1767318382.548569:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982545792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318382.548604:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318382.554028:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982545792:12345-0@lo:38:kworker.0 Request processed in 5448us (13855us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318382.554052:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 619 00000100:00100000:0.0:1767318382.554105:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318382.554225:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318382.554300:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469c700 x1853163982545664/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318398 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318382.554315:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982545664:0@lo:400:kworker.0 00000100:00080000:1.0:1767318382.561336:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318382.561347:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318382.561352:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982546560:0@lo:38:kworker.0 00000100:00080000:1.0:1767318382.561391:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318382.561396:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318382.561399:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982545792:0@lo:38:kworker.0 00000100:00080000:1.0:1767318382.561411:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318382.561415:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318382.561418:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982546048:0@lo:38:kworker.0 00000100:00080000:1.0:1767318382.561429:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318382.561433:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318382.561436:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982546176:0@lo:38:kworker.0 00010000:00010000:1.0:1767318383.685628:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318384.072095:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982546816 00000100:00100000:0.0:1767318384.072104:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982546816 00000100:00100000:0.0:1767318384.072178:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 346 00000100:00100000:0.0:1767318384.072187:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982546816:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318384.072218:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318384.072265:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982546816:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (617us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318384.072292:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 346 00000100:00080000:1.0:1767318384.072455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318384.073377:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318384.710861:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318385.547652:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318385.547694:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:1.0:1767318385.548195:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318386.756531:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318387.652807:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982547072 00000100:00100000:0.0:1767318387.652815:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982547072 00000100:00100000:0.0:1767318387.652856:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 623 00000100:00100000:0.0:1767318387.652863:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982547072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318387.652882:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318387.652971:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318387.652984:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982546944:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.653014:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318387.653074:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982546944 00000100:00100000:1.0:1767318387.653078:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982546944 00000100:00100000:1.0:1767318387.653132:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 483 00000100:00100000:1.0:1767318387.653140:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982546944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.653191:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982546944:12345-0@lo:400:kworker.0 Request processed in 42us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318387.653199:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 483 00000100:00080000:1.0:1767318387.653221:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318387.653248:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396df80 x1853163982546944/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318403 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318387.653261:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982546944:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.653577:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982547200:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.653603:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318387.653631:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982547200 00000100:00100000:1.0:1767318387.653635:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982547200 00000100:00100000:1.0:1767318387.653664:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 484 00000100:00100000:1.0:1767318387.653671:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982547200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.653703:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982547200:12345-0@lo:400:kworker.0 Request processed in 32us (98us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318387.653708:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 484 00000100:00080000:1.0:1767318387.653721:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318387.653737:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396df80 x1853163982547200/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318403 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318387.653746:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982547200:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.654815:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982547328 00000100:00100000:1.0:1767318387.654821:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982547328 00000100:00100000:1.0:1767318387.654850:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 624 00000100:00100000:1.0:1767318387.654857:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982547328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318387.654879:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318387.654914:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982547328:12345-0@lo:38:kworker.0 Request processed in 56us (182us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318387.654920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 624 00000100:00100000:0.0:1767318387.655251:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982547072:12345-0@lo:38:kworker.0 Request processed in 2386us (2556us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318387.655261:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 623 00000100:00100000:1.0:1767318387.655578:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982547712:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.655616:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318387.655649:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318387.655654:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982547584:0@lo:400:kworker.0 00000100:00100000:1.0:1767318387.655676:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318387.657771:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318387.657781:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318387.668399:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982547840 00000100:00100000:1.0:1767318387.668415:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982547840 00000100:00100000:1.0:1767318387.668475:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 625 00000100:00100000:1.0:1767318387.668494:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982547840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318387.668519:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318387.668613:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982547840:12345-0@lo:38:kworker.0 Request processed in 128us (2947us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318387.668621:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 625 00000100:00100000:0.0:1767318387.668768:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982547456 00000100:00100000:0.0:1767318387.668774:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982547456 00000100:00100000:0.0:1767318387.668799:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 181 00000100:00100000:0.0:1767318387.668805:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982547456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318387.668816:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318387.668847:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982547456:12345-0@lo:38:kworker.0 Request processed in 42us (3323us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318387.668853:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 181 00010000:00010000:1.0:1767318387.783713:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318388.677456:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460469f800 x1853163982540416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318387 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318388.677492:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318388.677529:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318388.677533:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982540416:0@lo:250:kworker.0 00000100:00100000:1.0:1767318388.678044:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982548096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318388.678120:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982548736:0@lo:400:kworker.0 00000100:00100000:2.0:1767318388.678205:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982547968:0@lo:250:kworker.0 00000100:00100000:1.0:1767318388.681952:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318388.682031:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318388.682043:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982548352:0@lo:400:kworker.0 00000100:00100000:1.0:1767318388.682080:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318388.682120:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548096 00000100:00100000:1.0:1767318388.682128:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548096 00000100:00100000:1.0:1767318388.682174:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548352 00000100:00100000:1.0:1767318388.682177:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548352 00000100:00100000:1.0:1767318388.682198:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 485 00000100:00100000:1.0:1767318388.682206:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982548096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318388.682326:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982548096:12345-0@lo:400:kworker.0 Request processed in 120us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318388.682334:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 485 00000100:00100000:1.0:1767318388.682352:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 486 00000100:00100000:1.0:1767318388.682358:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982548352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318388.682385:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982548352:12345-0@lo:400:kworker.0 Request processed in 28us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318388.682390:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 486 00010000:00080000:1.0:1767318388.682600:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469f800 x1853163982548352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318404 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318388.682626:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982548352:0@lo:400:kworker.0 00010000:00080000:1.0:1767318388.682639:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ce00 x1853163982548096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318404 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318388.682648:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982548096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318388.682863:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318388.685842:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548736 00000100:00100000:1.0:1767318388.685853:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548736 00000400:00000100:2.0:1767318388.685966:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982547968 offset 0 length 520: 4 00000100:00100000:2.0:1767318388.685991:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982548224:0@lo:38:kworker.0 00000100:00100000:2.0:1767318388.686024:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318388.686056:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982548480:0@lo:38:kworker.0 00000100:00100000:2.0:1767318388.686077:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318388.686096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982548608:0@lo:38:kworker.0 00000100:00100000:2.0:1767318388.686122:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318388.686141:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982548992:0@lo:38:kworker.0 00000100:00100000:0.0:1767318388.686161:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548864 00000100:00100000:0.0:1767318388.686165:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548864 00000100:00100000:2.0:1767318388.686171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318388.686196:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 327 00000100:00100000:2.0:1767318388.686201:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548224 00000100:00100000:2.0:1767318388.686205:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548224 00000100:00100000:0.0:1767318388.686206:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982548736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318388.686223:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548480 00000100:00100000:2.0:1767318388.686226:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548480 00000100:00100000:2.0:1767318388.686249:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548992 00000100:00100000:2.0:1767318388.686253:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548992 00000100:00100000:2.0:1767318388.686267:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 626 00000100:00100000:2.0:1767318388.686289:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548224:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318388.686309:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318388.686341:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982548736:12345-0@lo:400:kworker.0 Request processed in 136us (3477us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318388.686341:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548224:12345-0@lo:38:kworker.0 Request processed in 50us (315us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318388.686349:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 327 00000100:00100000:2.0:1767318388.686350:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 626 00000100:00100000:0.0:1767318388.686364:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 328 00000100:00100000:2.0:1767318388.686364:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 627 00000100:00100000:2.0:1767318388.686368:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548480:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767318388.686370:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982548864:12345-0@lo:400:kworker.0 00010000:00020000:2.0:1767318388.686377:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318388.686398:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548480:12345-0@lo:38:kworker.0 Request processed in 28us (320us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318388.686404:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 627 00000100:00100000:2.0:1767318388.686414:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 628 00000100:00100000:2.0:1767318388.686418:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318388.686427:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318388.686448:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982548992:12345-0@lo:38:kworker.0 Request processed in 29us (287us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318388.686453:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 628 00000100:00100000:2.0:1767318388.686480:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982548608 00000100:00100000:2.0:1767318388.686484:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982548608 00000100:00100000:2.0:1767318388.686505:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 182 00000100:00100000:2.0:1767318388.686509:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982548608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318388.686518:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318388.686543:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982548608:12345-0@lo:38:kworker.0 Request processed in 32us (420us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318388.686549:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 182 00000100:00080000:2.0:1767318388.686592:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318388.686597:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318388.686601:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982548224:0@lo:38:kworker.0 00000100:00080000:2.0:1767318388.686611:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318388.686615:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318388.686618:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982548480:0@lo:38:kworker.0 00000100:00080000:2.0:1767318388.686625:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318388.686628:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318388.686631:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982548608:0@lo:38:kworker.0 00000100:00080000:2.0:1767318388.686638:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318388.686641:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318388.686643:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982548992:0@lo:38:kworker.0 00000100:00100000:0.0:1767318388.686749:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982548864:12345-0@lo:400:kworker.0 Request processed in 378us (916us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318388.686756:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 328 00000100:00100000:0.0:1767318388.686796:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982548736:0@lo:400:kworker.0 00000100:00100000:1.0:1767318389.191890:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982549120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318389.191975:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318389.192208:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982549120 00000100:00100000:2.0:1767318389.192217:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982549120 00000100:00100000:2.0:1767318389.192302:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 348 00000100:00100000:2.0:1767318389.192313:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982549120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318389.192343:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767318389.192419:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318389.193738:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982549120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1423us (1759us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318389.193748:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 348 00000100:00100000:1.0:1767318389.193868:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982549120:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318389.828551:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318390.671990:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318390.672032:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318390.672112:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915565824 00000100:00100000:1.0:1767318390.672118:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915565824 00000100:00100000:1.0:1767318390.672170:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 629 00000100:00100000:1.0:1767318390.672179:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915565824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318390.672629:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318390.672671:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915565824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 492us (637us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318390.672678:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 629 00000100:00080000:2.0:1767318390.673161:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318390.853760:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318392.900834:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318393.797677:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318393.797696:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982549376:0@lo:400:kworker.0 00000100:00100000:1.0:1767318393.797844:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982549504:0@lo:38:kworker.0 00000100:00100000:1.0:1767318393.797924:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318393.797986:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982549760:0@lo:38:kworker.0 00000100:00100000:1.0:1767318393.798036:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318393.798082:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982549888:0@lo:38:kworker.0 00000100:00100000:2.0:1767318393.798465:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318393.798513:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982550144:0@lo:400:kworker.0 00000100:00100000:1.0:1767318393.798657:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318393.798710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982549632:0@lo:400:kworker.0 00000100:00100000:2.0:1767318393.798863:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318393.798976:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982549888 00000100:00100000:2.0:1767318393.798982:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982549888 00000100:00100000:2.0:1767318393.799014:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 183 00000100:00100000:2.0:1767318393.799024:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982549888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318393.799044:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318393.799509:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982549888:12345-0@lo:38:kworker.0 Request processed in 484us (849us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318393.799519:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 183 00000100:00100000:2.0:1767318393.799677:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 631 00000100:00100000:2.0:1767318393.799685:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982549760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318393.799698:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318393.799823:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982549760:12345-0@lo:38:kworker.0 Request processed in 136us (1788us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318393.799835:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 631 00000100:00080000:2.0:1767318393.799860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318393.799993:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550016 00000100:00100000:2.0:1767318393.800000:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550016 00000100:00100000:2.0:1767318393.800048:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 329 00000100:00100000:2.0:1767318393.800058:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982550016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318393.800664:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982550016:12345-0@lo:400:kworker.0 Request processed in 607us (2565us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318393.800673:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 329 00000100:00100000:0.0:1767318393.803981:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318393.805088:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318393.805098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318393.805102:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982549888:0@lo:38:kworker.0 00000100:00080000:1.0:1767318393.805116:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318393.805120:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318393.805123:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982549504:0@lo:38:kworker.0 00000100:00100000:1.0:1767318393.805153:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982550272:0@lo:38:kworker.0 00000100:00100000:1.0:1767318393.805200:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318393.805226:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318393.805230:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318393.805233:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982549760:0@lo:38:kworker.0 00000100:00100000:2.0:1767318393.805491:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550144 00000100:00100000:2.0:1767318393.805499:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550144 00000100:00100000:2.0:1767318393.805559:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 330 00000100:00100000:2.0:1767318393.805568:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982550144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318393.805629:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982550144:12345-0@lo:400:kworker.0 Request processed in 63us (1656us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318393.805639:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 330 00000100:00080000:0.0:1767318393.805955:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318393.806065:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982550144:0@lo:400:kworker.0 00000100:00080000:2.0:1767318393.806590:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318393.806595:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318393.806598:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982550272:0@lo:38:kworker.0 00010000:00010000:1.0:1767318393.925013:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318394.312205:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550400 00000100:00100000:0.0:1767318394.312213:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550400 00000100:00100000:0.0:1767318394.312259:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550528 00000100:00100000:0.0:1767318394.312262:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550528 00000100:00100000:0.0:1767318394.312313:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 350 00000100:00100000:0.0:1767318394.312323:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982550400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318394.312382:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318394.312424:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 351 00000100:00100000:0.0:1767318394.312432:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982550400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 109us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318394.312439:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982550528:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318394.312444:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 350 00002000:00000024:1.0:1767318394.312489:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318394.312527:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982550528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (433us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318394.312534:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 351 00000100:00100000:0.0:1767318395.798130:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318395.799027:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915566208 00000100:00100000:0.0:1767318395.799033:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915566208 00000100:00100000:0.0:1767318395.799087:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 490 00000100:00100000:0.0:1767318395.799095:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915566208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318395.799151:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915566208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 56us (191us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318395.799159:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 490 00010000:00010000:1.0:1767318395.972783:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318396.996916:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318398.917131:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982550656:0@lo:400:kworker.0 00000100:00100000:1.0:1767318398.917213:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318398.917304:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550656 00000100:00100000:2.0:1767318398.917321:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550656 00000100:00100000:2.0:1767318398.917401:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 491 00000100:00100000:2.0:1767318398.917412:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982550656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318398.917482:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982550656:12345-0@lo:400:kworker.0 Request processed in 73us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318398.917490:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 491 00000100:00080000:1.0:1767318398.917535:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318398.917560:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff100 x1853163982550656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318414 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318398.917569:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982550656:0@lo:400:kworker.0 00000100:00100000:1.0:1767318398.917683:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318398.917738:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982550784:0@lo:38:kworker.0 00000100:00100000:2.0:1767318398.917767:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318398.917838:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550784 00000100:00100000:0.0:1767318398.917843:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550784 00000100:00100000:0.0:1767318398.917874:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 634 00000100:00100000:0.0:1767318398.917878:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982550784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318398.917893:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318398.917919:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982550784:12345-0@lo:38:kworker.0 Request processed in 41us (152us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318398.917923:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 634 00000100:00080000:2.0:1767318398.918025:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318398.918028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318398.918031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982550784:0@lo:38:kworker.0 00000100:00100000:0.0:1767318398.918289:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982551296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318398.918328:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318398.918433:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551296 00000100:00100000:2.0:1767318398.918437:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551296 00000100:00100000:2.0:1767318398.918481:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551424 00000100:00100000:2.0:1767318398.918485:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551424 00000100:00100000:2.0:1767318398.918505:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 331 00000100:00100000:2.0:1767318398.918512:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982551296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318398.918554:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 332 00000100:00100000:2.0:1767318398.918560:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982551296:12345-0@lo:400:kworker.0 Request processed in 47us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318398.918564:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982551424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318398.918575:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 331 00000100:00100000:2.0:1767318398.918604:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982551040:0@lo:38:kworker.0 00000100:00100000:0.0:1767318398.918689:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982551424:12345-0@lo:400:kworker.0 Request processed in 128us (341us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318398.918696:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 332 00000100:00100000:1.0:1767318398.919633:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982550912:0@lo:400:kworker.0 00000100:00100000:2.0:1767318398.920774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318398.920813:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982551168:0@lo:38:kworker.0 00000100:00100000:2.0:1767318398.920845:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318398.920883:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982551552:0@lo:38:kworker.0 00000100:00100000:2.0:1767318398.920904:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318398.920954:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551040 00000100:00100000:0.0:1767318398.920962:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551040 00000100:00100000:0.0:1767318398.920993:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551552 00000100:00100000:0.0:1767318398.920996:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551552 00000100:00100000:0.0:1767318398.921014:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 635 00000100:00100000:0.0:1767318398.921022:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982551040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318398.921041:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318398.921187:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551168 00000100:00100000:2.0:1767318398.921192:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551168 00000100:00100000:2.0:1767318398.921214:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 184 00000100:00100000:2.0:1767318398.921218:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982551168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318398.921229:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318398.922562:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318398.922628:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982550912 00000100:00100000:1.0:1767318398.922635:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982550912 00000100:00100000:1.0:1767318398.923513:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 492 00000100:00100000:1.0:1767318398.923528:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982550912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318398.923609:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982550912:12345-0@lo:400:kworker.0 Request processed in 87us (1052us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318398.923617:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 492 00000100:00080000:1.0:1767318398.923850:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318398.923889:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff100 x1853163982550912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318414 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318398.923904:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982550912:0@lo:400:kworker.0 00000100:00100000:1.0:1767318398.924099:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982551296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318398.926664:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982551040:12345-0@lo:38:kworker.0 Request processed in 5631us (5883us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318398.926680:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 635 00000100:00080000:1.0:1767318398.926726:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318398.926735:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318398.926739:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982551040:0@lo:38:kworker.0 00000100:00100000:2.0:1767318398.926856:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982551168:12345-0@lo:38:kworker.0 Request processed in 5634us (6007us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318398.926866:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 184 00000100:00080000:1.0:1767318398.927017:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318398.927021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318398.927024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982551168:0@lo:38:kworker.0 00000100:00080000:1.0:1767318398.927034:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318398.927038:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318398.927041:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982551552:0@lo:38:kworker.0 00010000:00010000:1.0:1767318399.044497:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318399.428586:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551680 00000100:00100000:1.0:1767318399.428596:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551680 00000100:00100000:1.0:1767318399.428663:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 352 00000100:00100000:1.0:1767318399.428674:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982551680:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318399.428703:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00002000:00000024:1.0:1767318399.428704:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318399.428716:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982551808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318399.428749:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318399.428769:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982551680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318399.428783:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 352 00000100:00100000:0.0:1767318399.428791:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982551808 00000100:00100000:0.0:1767318399.428797:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982551808 00000100:00100000:0.0:1767318399.428824:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 353 00000100:00100000:0.0:1767318399.428833:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982551808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318399.428850:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318399.428899:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982551808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318399.428906:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 353 00000100:00100000:1.0:1767318399.429079:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982551808:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318400.068582:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318400.921420:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915566464 00000100:00100000:2.0:1767318400.921430:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915566464 00000100:00100000:2.0:1767318400.921494:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915566336 00000100:00100000:2.0:1767318400.921497:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915566336 00000100:00100000:2.0:1767318400.921513:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 493 00000100:00100000:2.0:1767318400.921522:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915566464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318400.921588:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915566464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 67us (360us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318400.921595:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 493 00000100:00100000:2.0:1767318400.921611:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 637 00000100:00100000:2.0:1767318400.921615:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915566336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318400.921633:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318400.921656:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915566336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 39us (319us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318400.921662:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 637 00010000:00010000:1.0:1767318402.118745:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318403.143521:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318404.036643:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318404.036654:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318404.036727:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318404.036740:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318404.036744:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318404.036753:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3647 00000100:00080000:0.0:1767318404.036760:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318404.036768:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318404.036796:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982552064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318404.036853:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318404.036857:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318404.036858:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318404.036862:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3647 00000100:00080000:0.0:1767318404.036865:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318404.036868:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318404.036879:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982552320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318404.036892:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318404.036895:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318404.036897:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318404.036899:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3647 00000100:00080000:0.0:1767318404.036902:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318404.036906:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318404.036916:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb480 x1853163982552448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318404.038319:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982552064:0@lo:38:kworker.0 00000100:00100000:2.0:1767318404.038356:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318404.038379:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982552320:0@lo:38:kworker.0 00000100:00100000:2.0:1767318404.038409:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318404.038427:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982552448:0@lo:38:kworker.0 00000100:00100000:2.0:1767318404.038450:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318404.038485:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982552448 00000100:00100000:2.0:1767318404.038489:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982552448 00000100:00100000:2.0:1767318404.038509:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 185 00000100:00100000:2.0:1767318404.038514:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982552448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318404.038550:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318404.038585:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982552448:12345-0@lo:38:kworker.0 Request processed in 69us (134us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318404.038592:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 185 00000100:00100000:2.0:1767318404.038609:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982552064 00000100:00100000:2.0:1767318404.038613:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982552064 00000100:00100000:2.0:1767318404.038628:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982552320 00000100:00100000:2.0:1767318404.038631:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982552320 00000100:00100000:2.0:1767318404.038643:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 638 00000100:00100000:2.0:1767318404.038647:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982552064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318404.038656:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318404.038679:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982552064:12345-0@lo:38:kworker.0 Request processed in 31us (323us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318404.038684:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 638 00000100:00100000:2.0:1767318404.038694:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 639 00000100:00100000:2.0:1767318404.038698:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982552320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318404.038707:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318404.038727:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982552320:12345-0@lo:38:kworker.0 Request processed in 27us (318us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318404.038733:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 639 00000100:00080000:2.0:1767318404.038761:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318404.038766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318404.038769:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982552064:0@lo:38:kworker.0 00000100:00080000:2.0:1767318404.038777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318404.038780:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318404.038782:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982552320:0@lo:38:kworker.0 00000100:00080000:2.0:1767318404.038789:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318404.038791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318404.038811:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982552448:0@lo:38:kworker.0 00000100:00080000:0.0:1767318404.038959:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318404.038965:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318404.038967:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318404.038972:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3647 00000100:00080000:0.0:1767318404.038975:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318404.038980:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318404.039002:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982552832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318404.039571:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982552832:0@lo:38:kworker.0 00000100:00100000:2.0:1767318404.039608:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318404.039899:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318404.039904:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318404.039907:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982552832:0@lo:38:kworker.0 00000100:00100000:1.0:1767318404.548553:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982552960:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318404.548660:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318404.548923:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982552960:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318404.552404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982553088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318404.552480:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318404.552623:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553088 00000100:00100000:0.0:1767318404.552631:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553088 00000100:00100000:0.0:1767318404.552739:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 355 00000100:00100000:0.0:1767318404.552749:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982553088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318404.552779:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318404.552824:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982553088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (344us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318404.552830:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 355 00000100:00100000:1.0:1767318404.552999:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982553088:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318405.189119:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318406.027547:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767318406.031012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318406.034081:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915566592 00000100:00100000:2.0:1767318406.034091:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915566592 00000100:00100000:2.0:1767318406.034145:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 641 00000100:00100000:2.0:1767318406.034154:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915566592:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318406.034176:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318406.034248:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915566592:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 93us (474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318406.034256:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 641 00010000:00010000:1.0:1767318406.216528:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318408.260543:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318409.157400:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982553344:0@lo:38:kworker.0 00000100:00100000:2.0:1767318409.158000:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318409.158206:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982553472:0@lo:400:kworker.0 00000100:00100000:0.0:1767318409.158239:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318409.158265:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318409.158291:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982553216:0@lo:400:kworker.0 00000100:00100000:0.0:1767318409.158313:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318409.158348:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553344 00000100:00100000:0.0:1767318409.158356:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553344 00000100:00100000:0.0:1767318409.158386:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553472 00000100:00100000:0.0:1767318409.158390:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553472 00000100:00100000:0.0:1767318409.158439:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553216 00000100:00100000:0.0:1767318409.158442:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553216 00000100:00100000:0.0:1767318409.158482:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 497 00000100:00100000:0.0:1767318409.158491:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982553472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318409.158560:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982553472:12345-0@lo:400:kworker.0 Request processed in 69us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318409.158568:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 497 00000100:00100000:0.0:1767318409.158585:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 498 00000100:00100000:0.0:1767318409.158592:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982553216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318409.158620:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982553216:12345-0@lo:400:kworker.0 Request processed in 28us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318409.158626:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 498 00000100:00100000:0.0:1767318409.158638:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 642 00000100:00100000:0.0:1767318409.158643:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982553344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318409.158665:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767318409.158781:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318409.158827:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cdf80 x1853163982553216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318425 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318409.158841:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982553216:0@lo:400:kworker.0 00010000:00080000:1.0:1767318409.158853:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55c00 x1853163982553472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318425 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318409.158861:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982553472:0@lo:400:kworker.0 00000100:00100000:2.0:1767318409.160041:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982553600:0@lo:38:kworker.0 00000100:00100000:2.0:1767318409.160107:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318409.162049:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982553344:12345-0@lo:38:kworker.0 Request processed in 3402us (4049us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318409.162058:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 642 00000100:00100000:0.0:1767318409.162085:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553600 00000100:00100000:0.0:1767318409.162090:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553600 00000100:00100000:1.0:1767318409.162982:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 643 00000100:00100000:1.0:1767318409.162990:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982553600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318409.163014:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318409.163082:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982553600:12345-0@lo:38:kworker.0 Request processed in 80us (2963us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318409.163089:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 643 00000100:00080000:2.0:1767318409.163195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318409.163202:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318409.163206:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982553344:0@lo:38:kworker.0 00000100:00100000:2.0:1767318409.163339:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982553728:0@lo:38:kworker.0 00000100:00100000:2.0:1767318409.163375:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318409.163483:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982554112:0@lo:38:kworker.0 00000100:00100000:2.0:1767318409.163506:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318409.164689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318409.164694:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318409.164697:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982554112:0@lo:38:kworker.0 00000100:00080000:2.0:1767318409.164720:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318409.164724:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318409.164728:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982553600:0@lo:38:kworker.0 00000100:00100000:1.0:1767318409.165218:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982553728 00000100:00100000:1.0:1767318409.165225:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982553728 00000100:00100000:1.0:1767318409.165407:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 186 00000100:00100000:1.0:1767318409.165414:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982553728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318409.165955:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318409.166087:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982553728:12345-0@lo:38:kworker.0 Request processed in 672us (2710us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318409.166095:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 186 00000100:00080000:2.0:1767318409.166160:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318409.166165:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318409.166169:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982553728:0@lo:38:kworker.0 00010000:00010000:1.0:1767318409.284530:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318409.668633:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318409.668659:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982554240:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318409.668729:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318409.669100:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982554240 00000100:00100000:0.0:1767318409.669107:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982554240 00000100:00100000:0.0:1767318409.669153:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982554368 00000100:00100000:0.0:1767318409.669155:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982554368 00000100:00100000:0.0:1767318409.669173:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 356 00000100:00100000:0.0:1767318409.669180:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982554240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318409.669216:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318409.669256:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982554240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (527us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318409.669263:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 356 00000100:00100000:0.0:1767318409.669303:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 357 00000100:00100000:0.0:1767318409.669309:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982554368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318409.669323:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318409.669350:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982554368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (332us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318409.669355:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 357 00000100:00100000:0.0:1767318409.669391:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982554368:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318409.669739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982554240:0@lo:13:osp-pre-1-1.0 00000400:00000100:0.0:1767318410.126209:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915566848 offset 0 length 520: 4 00000100:00100000:0.0:1767318410.126489:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318410.126562:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318410.134832:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915566976 00000100:00100000:2.0:1767318410.134852:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915566976 00000100:00100000:2.0:1767318410.134907:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567104 00000100:00100000:2.0:1767318410.134910:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567104 00000100:00100000:2.0:1767318410.134966:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 499 00000100:00100000:2.0:1767318410.134977:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318410.135097:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 645 00000100:00100000:1.0:1767318410.135106:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915566976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318410.135129:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318410.135450:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318410.138708:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 3721us (12132us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318410.138722:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 499 00000100:00100000:1.0:1767318410.138966:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915566976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3859us (12476us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318410.138975:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 645 00010000:00010000:1.0:1767318411.332583:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318412.356730:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318413.252473:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318388/real 1767318388] req@ffff93460469c700 x1853163982547968/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318413 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318413.252534:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460469c700 x1853163982547968/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318413 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318413.252565:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318413.252591:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318413.252595:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982547968:0@lo:250:kworker.0 00000100:00100000:1.0:1767318413.253419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982554880:0@lo:400:kworker.0 00000100:00100000:1.0:1767318413.253532:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318413.253627:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318413.253634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982554624:0@lo:400:kworker.0 00000100:00100000:1.0:1767318413.253657:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318413.253913:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982554624 00000100:00100000:2.0:1767318413.253917:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982554624 00000100:00080000:1.0:1767318413.254849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318413.254854:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318413.254866:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982555136 00000100:00100000:1.0:1767318413.254869:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982555136 00000100:00100000:1.0:1767318413.254905:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 187 00000100:00100000:1.0:1767318413.254909:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982555136:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318413.254918:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318413.254947:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982555136:12345-0@lo:38:kworker.0 Request processed in 37us (611us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318413.254953:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 187 00010000:00080000:1.0:1767318413.254981:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aad80 x1853163982554880/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318429 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318413.255004:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982554880:0@lo:400:kworker.0 00000100:00080000:1.0:1767318413.255025:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318413.255029:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318413.255032:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982555136:0@lo:38:kworker.0 00000100:00100000:1.0:1767318413.255064:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982555264 00000100:00100000:1.0:1767318413.255068:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982555264 00000100:00100000:1.0:1767318413.255100:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 337 00000100:00100000:1.0:1767318413.255107:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982555264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318413.255144:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982555264:12345-0@lo:400:kworker.0 Request processed in 38us (365us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318413.255150:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 337 00000100:00100000:1.0:1767318413.255177:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982555264:0@lo:400:kworker.0 00000100:00080000:1.0:1767318413.255193:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318413.258286:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 501 00000100:00100000:2.0:1767318413.258301:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982554624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318413.258413:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982554624:12345-0@lo:400:kworker.0 Request processed in 116us (4754us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318413.258436:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 501 00010000:00080000:1.0:1767318413.259191:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a8000 x1853163982554624/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318429 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318413.259209:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982554624:0@lo:400:kworker.0 00000100:00100000:2.0:1767318413.259814:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982555392:0@lo:400:kworker.0 00010000:00010000:1.0:1767318414.407350:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318414.788507:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318414.788531:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982555648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318414.788603:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318414.788698:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982555648 00000100:00100000:1.0:1767318414.788705:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982555648 00000100:00100000:1.0:1767318414.788759:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 358 00000100:00100000:1.0:1767318414.788768:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982555648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318414.788846:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318414.788908:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982555648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318414.788915:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 358 00000100:00100000:2.0:1767318414.789006:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982555648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318414.789758:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982555776:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318414.789791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318414.789852:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982555776 00000100:00100000:0.0:1767318414.789858:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982555776 00000100:00100000:0.0:1767318414.789889:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 359 00000100:00100000:0.0:1767318414.789897:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982555776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318414.789915:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318414.793029:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982555776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3122us (3226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318414.793045:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 359 00000100:00100000:2.0:1767318414.793856:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982555776:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318415.244366:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567232 00000100:00100000:0.0:1767318415.244386:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567232 00000100:00100000:0.0:1767318415.244433:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567360 00000100:00100000:0.0:1767318415.244436:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567360 00000100:00100000:0.0:1767318415.244489:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 502 00000100:00100000:0.0:1767318415.244500:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318415.244528:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318415.244583:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318415.244603:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 502 00010000:00010000:1.0:1767318415.430778:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318417.476846:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318418.373313:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318418.373325:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318418.373471:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318418.373475:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318418.373479:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318418.373487:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3661 00000100:00080000:0.0:1767318418.373493:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318418.373501:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318418.373518:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982555904:0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.373529:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982556032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318418.373567:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318418.373571:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318418.373573:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318418.373576:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3661 00000100:00080000:0.0:1767318418.373580:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318418.373583:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318418.373598:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982556288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318418.373599:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318418.373611:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318418.373615:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318418.373618:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318418.373622:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3661 00000100:00080000:0.0:1767318418.373625:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318418.373629:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318418.373642:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982556416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318418.373647:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318418.373657:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982556160:0@lo:400:kworker.0 00000100:00080000:0.0:1767318418.373680:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318418.373685:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318418.373687:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767318418.373693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318418.373695:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3661 00000100:00080000:0.0:1767318418.373699:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318418.373704:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318418.373717:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb480 x1853163982556800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318418.373738:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982555904 00000100:00100000:2.0:1767318418.373748:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982555904 00000100:00100000:0.0:1767318418.373764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982556672:0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.373806:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318418.373818:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982556160 00000100:00100000:2.0:1767318418.373823:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982556160 00000100:00100000:0.0:1767318418.373843:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318418.373851:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982556544:0@lo:400:kworker.0 00000100:00100000:2.0:1767318418.373862:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 503 00000100:00100000:2.0:1767318418.373874:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982555904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.373882:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318418.373927:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 504 00000100:00100000:0.0:1767318418.373940:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982556160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318418.373961:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982555904:12345-0@lo:400:kworker.0 Request processed in 87us (362us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318418.373971:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 503 00000100:00100000:0.0:1767318418.373985:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982556160:12345-0@lo:400:kworker.0 Request processed in 49us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318418.373992:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 504 00000100:00100000:0.0:1767318418.374022:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982556672 00000100:00100000:0.0:1767318418.374026:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982556672 00010000:00080000:2.0:1767318418.374037:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9880 x1853163982556160/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318434 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318418.374053:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982556160:0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.374059:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982556544 00000100:00100000:0.0:1767318418.374063:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982556544 00010000:00080000:2.0:1767318418.374084:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fad80 x1853163982555904/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318434 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318418.374087:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 339 00000100:00100000:0.0:1767318418.374098:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982556672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318418.374101:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982555904:0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.374149:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982556672:12345-0@lo:400:kworker.0 Request processed in 52us (341us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318418.374156:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 339 00000100:00100000:0.0:1767318418.374171:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 340 00000100:00100000:0.0:1767318418.374178:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982556544:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318418.374205:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982556544:12345-0@lo:400:kworker.0 Request processed in 28us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318418.374211:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 340 00000100:00080000:1.0:1767318418.374242:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318418.374245:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982556544:0@lo:400:kworker.0 00000100:00080000:1.0:1767318418.374250:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318418.374255:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318418.374260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982556672:0@lo:400:kworker.0 00000100:00100000:2.0:1767318418.374774:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 652 00000100:00100000:2.0:1767318418.374782:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982556800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318418.374794:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318418.374831:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982556800:12345-0@lo:38:kworker.0 Request processed in 48us (275us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318418.374841:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 652 00000100:00100000:1.0:1767318418.375509:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982556416 00000100:00100000:1.0:1767318418.375523:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982556416 00000100:00100000:1.0:1767318418.375592:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 188 00000100:00100000:1.0:1767318418.375599:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982556416:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318418.375628:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318418.375697:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982556416:12345-0@lo:38:kworker.0 Request processed in 97us (1188us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318418.375703:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 188 00010000:00010000:1.0:1767318418.500457:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318419.911024:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318419.914814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557056:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318419.914968:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318419.916359:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557056:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318420.367327:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567616 00000100:00100000:0.0:1767318420.367338:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567616 00000100:00080000:1.0:1767318420.367482:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318420.367536:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 505 00000100:00100000:0.0:1767318420.367554:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318420.367641:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915567616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (431us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318420.367649:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 505 00000100:00100000:0.0:1767318420.371285:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567488 00000100:00100000:0.0:1767318420.371302:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567488 00000100:00100000:1.0:1767318420.371500:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 653 00000100:00100000:1.0:1767318420.371509:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915567488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318420.371532:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318420.371606:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915567488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 96us (4081us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318420.371613:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 653 00010000:00010000:1.0:1767318420.549041:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318421.573646:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318423.492477:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318423.492487:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318423.492549:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318423.492553:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318423.492556:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318423.492564:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3666 00000100:00080000:0.0:1767318423.492569:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318423.492575:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318423.492597:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8e00 x1853163982557312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318423.492616:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318423.492620:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318423.492621:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318423.492625:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3666 00000100:00080000:0.0:1767318423.492628:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318423.492631:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318423.492642:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163982557568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318423.492650:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318423.492653:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318423.492654:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318423.492657:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3666 00000100:00080000:0.0:1767318423.492659:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318423.492663:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318423.492674:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982557696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318423.492700:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318423.492703:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318423.492705:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318423.492708:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3666 00000100:00080000:0.0:1767318423.492710:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318423.492712:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318423.492720:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982558080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318423.492781:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982557184:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.492853:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318423.492894:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318423.492901:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982557440:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.492922:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318423.492955:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557184 00000100:00100000:2.0:1767318423.492962:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557184 00000100:00100000:2.0:1767318423.493014:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557440 00000100:00100000:2.0:1767318423.493017:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557440 00000100:00100000:2.0:1767318423.493042:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 506 00000100:00100000:2.0:1767318423.493050:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982557184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.493097:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982557184:12345-0@lo:400:kworker.0 Request processed in 48us (244us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318423.493104:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 506 00000100:00100000:2.0:1767318423.493120:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 507 00000100:00100000:2.0:1767318423.493125:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982557440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.493162:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982557440:12345-0@lo:400:kworker.0 Request processed in 36us (238us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318423.493168:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 507 00000100:00080000:1.0:1767318423.500091:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318423.501719:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557312 00000100:00100000:2.0:1767318423.501729:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557312 00000100:00100000:2.0:1767318423.501758:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557568 00000100:00100000:2.0:1767318423.501760:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557568 00000100:00100000:2.0:1767318423.501768:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 654 00000100:00100000:2.0:1767318423.501774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982557312:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318423.501803:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318423.501860:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982557312:12345-0@lo:38:kworker.0 Request processed in 84us (8514us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318423.501867:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 654 00000100:00100000:2.0:1767318423.501881:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 655 00000100:00100000:2.0:1767318423.501884:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982557568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318423.502054:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318423.502077:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982557568:12345-0@lo:38:kworker.0 Request processed in 192us (1620us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318423.502081:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 655 00010000:00080000:2.0:1767318423.502123:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9c00 x1853163982557440/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318439 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318423.502136:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982557440:0@lo:400:kworker.0 00010000:00080000:2.0:1767318423.502158:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fbb80 x1853163982557184/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318439 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318423.502164:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982557184:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502191:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557824:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502218:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318423.502243:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318423.502250:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502265:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318423.502300:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557824 00000100:00100000:2.0:1767318423.502303:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557824 00000100:00100000:2.0:1767318423.502353:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557952 00000100:00100000:2.0:1767318423.502357:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557952 00000100:00100000:2.0:1767318423.502379:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 341 00000100:00100000:2.0:1767318423.502384:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982557824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502415:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982557824:12345-0@lo:400:kworker.0 Request processed in 31us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318423.502419:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 341 00000100:00100000:2.0:1767318423.502427:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 342 00000100:00100000:2.0:1767318423.502431:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982557952:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502817:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982557952:12345-0@lo:400:kworker.0 Request processed in 384us (548us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318423.502824:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 342 00000100:00100000:2.0:1767318423.502866:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318423.502874:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982557824:0@lo:400:kworker.0 00000100:00080000:2.0:1767318423.502941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318423.502944:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318423.502952:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982557696 00000100:00100000:2.0:1767318423.502955:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982557696 00000100:00100000:2.0:1767318423.502969:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 189 00000100:00100000:2.0:1767318423.502972:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982557696:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318423.502990:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318423.503009:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982557696:12345-0@lo:38:kworker.0 Request processed in 36us (1612us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318423.503013:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 189 00000100:00100000:1.0:1767318423.504288:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558080 00000100:00100000:1.0:1767318423.504297:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558080 00000100:00100000:1.0:1767318423.504649:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767318423.504656:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982558080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318423.504670:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318423.505290:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982558080:12345-0@lo:38:kworker.0 Request processed in 633us (3856us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318423.505295:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 656 00010000:00010000:1.0:1767318423.620525:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318424.649792:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318425.028890:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318425.028908:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982558208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318425.028971:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318425.029358:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318425.029416:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982558208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318425.030023:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982558336:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318425.030055:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318425.030173:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558336 00000100:00100000:2.0:1767318425.030179:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558336 00000100:00100000:2.0:1767318425.030222:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 363 00000100:00100000:2.0:1767318425.030229:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982558336:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318425.030263:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767318425.030501:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318425.031860:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982558336:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1623us (1795us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318425.031874:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 363 00000100:00100000:0.0:1767318425.031964:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982558336:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318425.486880:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318425.486988:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567744 00000100:00100000:2.0:1767318425.487002:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567744 00000100:00100000:2.0:1767318425.487056:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 657 00000100:00100000:2.0:1767318425.487066:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915567744:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318425.487106:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318425.487175:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915567744:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 109us (304us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318425.487183:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 657 00000100:00100000:2.0:1767318425.489042:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915567872 00000100:00100000:2.0:1767318425.489057:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915567872 00000100:00100000:2.0:1767318425.489144:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 508 00000100:00100000:2.0:1767318425.489155:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915567872:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318425.489162:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318425.489413:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915567872:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 258us (1623us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318425.489424:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 508 00010000:00010000:1.0:1767318426.694950:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318428.618602:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318428.618621:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318428.618853:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318428.618858:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318428.618862:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318428.618871:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3672 00000100:00080000:1.0:1767318428.618875:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318428.618925:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318428.619100:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982558464:0@lo:400:kworker.0 00000100:00100000:2.0:1767318428.619170:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318428.619371:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396df80 x1853163982558592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318428.619429:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982558592:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.619457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318428.619513:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318428.619516:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318428.619519:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318428.619523:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3672 00000100:00080000:1.0:1767318428.619525:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318428.619529:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318428.619542:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163982558848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318428.619808:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558464 00000100:00100000:0.0:1767318428.619818:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558464 00000100:00100000:0.0:1767318428.619874:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558592 00000100:00100000:0.0:1767318428.619877:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558592 00000100:00100000:0.0:1767318428.619895:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 509 00000100:00100000:0.0:1767318428.619907:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982558464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318428.619985:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982558464:12345-0@lo:400:kworker.0 Request processed in 81us (814us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318428.619993:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 509 00000100:00100000:0.0:1767318428.620012:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 658 00000100:00100000:0.0:1767318428.620016:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982558592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318428.620037:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318428.620555:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982558848:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.621079:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318428.623857:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318428.623868:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318428.623872:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318428.623878:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3672 00000100:00080000:1.0:1767318428.623882:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318428.623889:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318428.623928:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37800 x1853163982558976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318428.624010:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318428.624014:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318428.624016:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318428.624020:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3672 00000100:00080000:1.0:1767318428.624023:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318428.624027:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318428.624040:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163982559360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318428.624077:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982559104:0@lo:400:kworker.0 00000100:00100000:1.0:1767318428.624130:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318428.624156:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318428.624163:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982559232:0@lo:400:kworker.0 00000100:00100000:1.0:1767318428.624186:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318428.625543:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982558592:12345-0@lo:38:kworker.0 Request processed in 5519us (6080us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318428.625557:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 658 00000100:00100000:0.0:1767318428.625583:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558848 00000100:00100000:0.0:1767318428.625590:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558848 00000100:00080000:2.0:1767318428.626132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318428.626139:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318428.626142:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982558592:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.626169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982558976:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.626217:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318428.626263:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982559360:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.626302:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318428.626354:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982558976 00000100:00100000:1.0:1767318428.626360:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982558976 00000100:00100000:1.0:1767318428.626388:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 190 00000100:00100000:1.0:1767318428.626396:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982558976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318428.626416:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318428.626456:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982558976:12345-0@lo:38:kworker.0 Request processed in 60us (238us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318428.626463:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 190 00000100:00080000:2.0:1767318428.627177:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318428.627184:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318428.627191:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982558976:0@lo:38:kworker.0 00000100:00100000:2.0:1767318428.627222:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982559104 00000100:00100000:2.0:1767318428.627228:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982559104 00000100:00100000:2.0:1767318428.627331:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982559232 00000100:00100000:2.0:1767318428.627335:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982559232 00000100:00100000:2.0:1767318428.627357:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 343 00000100:00100000:2.0:1767318428.627363:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982559104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318428.627409:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982559104:12345-0@lo:400:kworker.0 Request processed in 45us (3279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318428.627416:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 343 00000100:00100000:2.0:1767318428.627428:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 344 00000100:00100000:2.0:1767318428.627433:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982559232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318428.627466:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982559232:12345-0@lo:400:kworker.0 Request processed in 33us (3279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318428.627473:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 344 00000100:00080000:2.0:1767318428.627508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318428.627513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318428.627561:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982559232:0@lo:400:kworker.0 00000100:00100000:2.0:1767318428.627573:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982559104:0@lo:400:kworker.0 00000100:00080000:2.0:1767318428.627598:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318428.627766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318428.627770:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982558848:0@lo:38:kworker.0 00000100:00080000:2.0:1767318428.627792:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318428.627795:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318428.627798:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982559360:0@lo:38:kworker.0 00010000:00010000:1.0:1767318429.765051:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318430.148648:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318430.148679:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982559488:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318430.148774:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318430.148964:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982559488 00000100:00100000:1.0:1767318430.148973:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982559488 00000100:00100000:1.0:1767318430.149066:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 364 00000100:00100000:1.0:1767318430.149077:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982559488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318430.149126:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318430.149184:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982559488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 108us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318430.149192:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 364 00000100:00080000:0.0:1767318430.149320:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318430.149362:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982559488:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318430.149544:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982559616:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318430.149587:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318430.149874:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318430.149899:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982559616:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318430.605715:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915568128 00000100:00100000:1.0:1767318430.605725:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915568128 00000100:00080000:0.0:1767318430.605775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318430.605799:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915568000 00000100:00100000:1.0:1767318430.605803:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915568000 00000100:00100000:1.0:1767318430.605841:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 661 00000100:00100000:1.0:1767318430.605846:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915568000:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318430.605865:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318430.611737:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915568000:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 5874us (6087us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318430.611755:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 661 00010000:00010000:1.0:1767318432.840157:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318433.732773:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318433.732783:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318433.732840:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318433.732844:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318433.732847:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318433.732856:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3677 00000100:00080000:0.0:1767318433.732861:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318433.732867:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318433.732947:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8e00 x1853163982559872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318433.732984:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318433.732988:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318433.732990:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318433.732993:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3677 00000100:00080000:0.0:1767318433.732996:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318433.733000:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318433.733012:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982560128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318433.733021:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318433.733025:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318433.733027:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318433.733030:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3677 00000100:00080000:0.0:1767318433.733033:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318433.733036:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318433.733048:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982560256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318433.733403:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982560000:0@lo:400:kworker.0 00000100:00100000:1.0:1767318433.733472:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318433.733576:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318433.733583:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982559744:0@lo:400:kworker.0 00000100:00100000:1.0:1767318433.733617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318433.733654:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560000 00000100:00100000:1.0:1767318433.733661:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560000 00000100:00100000:2.0:1767318433.733824:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982559744 00000100:00100000:2.0:1767318433.733829:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982559744 00000100:00080000:0.0:1767318433.738565:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318433.738572:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318433.738575:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318433.738583:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3677 00000100:00080000:0.0:1767318433.738588:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318433.738595:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318433.738632:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982560640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318433.738686:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982560384:0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.740638:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318433.740671:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318433.740680:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982560512:0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.740707:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318433.740759:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318433.741689:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982559872 00000100:00100000:1.0:1767318433.741700:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982559872 00000100:00100000:1.0:1767318433.741733:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560128 00000100:00100000:1.0:1767318433.741737:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560128 00000100:00100000:1.0:1767318433.741755:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560640 00000100:00100000:1.0:1767318433.741758:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560640 00000100:00100000:1.0:1767318433.741772:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 512 00000100:00100000:1.0:1767318433.741783:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982560000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.741973:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 513 00000100:00100000:0.0:1767318433.741984:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982559744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.742075:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982559744:12345-0@lo:400:kworker.0 Request processed in 93us (8457us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318433.742087:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 513 00000100:00100000:0.0:1767318433.742108:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 662 00000100:00100000:0.0:1767318433.742113:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982559872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318433.742135:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318433.742172:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982559872:12345-0@lo:38:kworker.0 Request processed in 57us (8205us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318433.742178:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 662 00000100:00100000:0.0:1767318433.742192:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 663 00000100:00100000:0.0:1767318433.742195:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982560128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318433.742204:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318433.742240:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982560128:12345-0@lo:38:kworker.0 Request processed in 28us (8213us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318433.742246:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 663 00000100:00100000:0.0:1767318433.742258:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 664 00000100:00100000:0.0:1767318433.742262:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982560640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318433.742286:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318433.742341:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982560640:12345-0@lo:38:kworker.0 Request processed in 77us (2605us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318433.742346:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 664 00000100:00100000:0.0:1767318433.742367:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560384 00000100:00100000:0.0:1767318433.742851:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560384 00000100:00100000:0.0:1767318433.742901:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560512 00000100:00100000:0.0:1767318433.742904:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560512 00000100:00100000:0.0:1767318433.742937:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 345 00000100:00100000:0.0:1767318433.742945:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982560384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.742988:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982560384:12345-0@lo:400:kworker.0 Request processed in 45us (2353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318433.742995:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 345 00000100:00100000:0.0:1767318433.743008:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 346 00000100:00100000:0.0:1767318433.743025:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982560512:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.743054:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982560512:12345-0@lo:400:kworker.0 Request processed in 40us (2345us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318433.743060:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 346 00000100:00080000:0.0:1767318433.743080:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318433.743083:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318433.743121:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982560512:0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.743133:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982560384:0@lo:400:kworker.0 00010000:00080000:0.0:1767318433.743186:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076faa00 x1853163982559744/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318449 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318433.743197:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982559744:0@lo:400:kworker.0 00000100:00100000:0.0:1767318433.743543:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560256 00000100:00100000:0.0:1767318433.743549:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560256 00000100:00100000:0.0:1767318433.743611:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 191 00000100:00100000:0.0:1767318433.743616:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982560256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318433.743633:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318433.743668:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982560256:12345-0@lo:38:kworker.0 Request processed in 50us (4035us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318433.743673:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 191 00000100:00080000:0.0:1767318433.743704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318433.743709:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318433.743712:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982560640:0@lo:38:kworker.0 00000100:00080000:0.0:1767318433.743721:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318433.743725:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318433.743728:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982559872:0@lo:38:kworker.0 00000100:00080000:0.0:1767318433.743736:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318433.743739:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318433.743741:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982560128:0@lo:38:kworker.0 00000100:00080000:0.0:1767318433.743853:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318433.743857:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318433.743860:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982560256:0@lo:38:kworker.0 00000100:00100000:1.0:1767318433.744260:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982560000:12345-0@lo:400:kworker.0 Request processed in 2481us (10788us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318433.744268:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 512 00010000:00080000:0.0:1767318433.744585:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb480 x1853163982560000/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318449 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318433.744606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982560000:0@lo:400:kworker.0 00000100:00100000:1.0:1767318434.702538:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318434.702639:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915568512 00000100:00100000:2.0:1767318434.702653:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915568512 00000100:00100000:2.0:1767318434.702794:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 514 00000100:00100000:2.0:1767318434.702806:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915568512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318434.702946:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915568512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 142us (418us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318434.702966:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 514 00000100:00080000:0.0:1767318434.704474:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767318434.706156:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915568256 offset 0 length 520: 4 00000100:00100000:0.0:1767318434.706205:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318435.269055:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560768 00000100:00100000:1.0:1767318435.269063:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560768 00000100:00100000:1.0:1767318435.269121:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 366 00000100:00100000:1.0:1767318435.269131:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982560768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318435.269163:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318435.269229:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982560768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (424us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318435.269237:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 366 00000100:00100000:2.0:1767318435.269524:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982560896 00000100:00100000:2.0:1767318435.269530:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982560896 00000100:00100000:2.0:1767318435.269564:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 367 00000100:00100000:2.0:1767318435.269582:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982560896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318435.269609:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318435.269650:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982560896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (250us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318435.269656:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 367 00000100:00080000:0.0:1767318435.269848:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318435.269853:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318435.908611:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318438.852447:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318413/real 1767318413] req@ffff93463e0a9880 x1853163982554496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318438 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318438.852481:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463e0a9880 x1853163982554496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318438 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318438.852529:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318438.852540:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318438.852543:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982554496:0@lo:250:kworker.0 00000100:00100000:2.0:1767318438.852774:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318438.852788:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982561152:0@lo:400:kworker.0 00000100:00100000:1.0:1767318438.852803:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318438.852819:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982561408:0@lo:400:kworker.0 00000100:00100000:2.0:1767318438.852850:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318438.852887:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318438.853022:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982561024:0@lo:250:kworker.0 00000400:00000100:1.0:1767318438.853059:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982561024 offset 0 length 520: 4 00000100:00100000:1.0:1767318438.853077:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982561280:0@lo:38:kworker.0 00000100:00100000:1.0:1767318438.853099:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318438.853115:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982561536:0@lo:38:kworker.0 00000100:00100000:1.0:1767318438.853133:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318438.858247:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982561792:0@lo:400:kworker.0 00000100:00100000:0.0:1767318438.860025:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318438.860103:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318438.860117:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982561920:0@lo:400:kworker.0 00000100:00100000:0.0:1767318438.860141:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318438.860187:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561792 00000100:00100000:0.0:1767318438.860196:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561792 00000100:00080000:0.0:1767318438.865410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318438.865418:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318438.865434:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561920 00000100:00100000:0.0:1767318438.865443:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561920 00000100:00100000:0.0:1767318438.865488:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 347 00000100:00100000:0.0:1767318438.865496:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982561920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318438.865594:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982561920:12345-0@lo:400:kworker.0 Request processed in 98us (5451us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318438.865603:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 347 00000100:00100000:2.0:1767318438.870089:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561408 00000100:00100000:2.0:1767318438.870107:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561408 00000100:00100000:2.0:1767318438.870339:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561280 00000100:00100000:2.0:1767318438.870344:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561280 00000100:00100000:2.0:1767318438.870354:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561536 00000100:00100000:2.0:1767318438.870357:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561536 00000100:00100000:2.0:1767318438.870372:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 515 00000100:00100000:2.0:1767318438.870384:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982561152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318438.871015:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982561152:12345-0@lo:400:kworker.0 Request processed in 632us (18162us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318438.871024:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 515 00000100:00100000:2.0:1767318438.871046:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 516 00000100:00100000:2.0:1767318438.871054:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982561408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318438.871096:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982561408:12345-0@lo:400:kworker.0 Request processed in 44us (18214us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318438.871102:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 516 00000100:00100000:2.0:1767318438.871114:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 666 00000100:00100000:2.0:1767318438.871118:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982561280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318438.871639:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318438.871674:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982561280:12345-0@lo:38:kworker.0 Request processed in 554us (18574us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318438.871681:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 666 00000100:00100000:2.0:1767318438.871694:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 667 00000100:00100000:2.0:1767318438.871698:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982561536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318438.871707:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318438.871729:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982561536:12345-0@lo:38:kworker.0 Request processed in 30us (18595us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318438.871734:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 667 00010000:00080000:2.0:1767318438.871786:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9880 x1853163982561408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318454 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318438.871800:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982561408:0@lo:400:kworker.0 00010000:00080000:2.0:1767318438.871837:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee7480 x1853163982561152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318454 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318438.872022:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318438.872031:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318438.872035:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982561536:0@lo:38:kworker.0 00000100:00100000:1.0:1767318438.872046:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982561664:0@lo:38:kworker.0 00000100:00100000:1.0:1767318438.872096:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318438.872133:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982562048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318438.872152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318438.872172:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562048 00000100:00100000:1.0:1767318438.872176:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562048 00000100:00100000:1.0:1767318438.872201:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 668 00000100:00100000:1.0:1767318438.872206:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982562048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318438.872217:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318438.872238:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982562048:12345-0@lo:38:kworker.0 Request processed in 31us (85us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318438.872243:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 668 00000100:00080000:1.0:1767318438.872263:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318438.872290:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318438.872293:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982562048:0@lo:38:kworker.0 00000100:00080000:1.0:1767318438.872317:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318438.872321:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318438.872324:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982561280:0@lo:38:kworker.0 00000100:00080000:1.0:1767318438.872370:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318438.872375:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318438.872406:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 348 00000100:00100000:1.0:1767318438.872414:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982561792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318438.872450:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982561792:12345-0@lo:400:kworker.0 Request processed in 38us (12428us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318438.872456:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 348 00000100:00100000:2.0:1767318438.872615:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982561152:0@lo:400:kworker.0 00000100:00100000:1.0:1767318438.872841:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982561920:0@lo:400:kworker.0 00000100:00100000:1.0:1767318438.872855:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982561792:0@lo:400:kworker.0 00000100:00100000:1.0:1767318438.872887:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982561664 00000100:00100000:1.0:1767318438.872891:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982561664 00000100:00100000:1.0:1767318438.872921:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 192 00000100:00100000:1.0:1767318438.872926:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982561664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318438.872937:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318438.872971:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982561664:12345-0@lo:38:kworker.0 Request processed in 44us (876us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318438.872978:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 192 00000100:00080000:1.0:1767318438.873006:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318438.873012:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318438.873015:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982561664:0@lo:38:kworker.0 00010000:00010000:1.0:1767318438.980772:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318439.820566:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:1.0:1767318439.821584:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318440.389508:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982562176:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318440.389596:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318440.389733:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318440.389741:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982562304:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318440.389766:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318440.389976:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562176 00000100:00100000:1.0:1767318440.389984:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562176 00000100:00100000:1.0:1767318440.390044:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562304 00000100:00100000:1.0:1767318440.390047:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562304 00000100:00100000:0.0:1767318440.390056:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 368 00000100:00100000:1.0:1767318440.390066:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 369 00000100:00100000:0.0:1767318440.390067:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982562176:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318440.390071:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982562304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318440.390095:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:1.0:1767318440.390101:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318440.390140:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982562176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (544us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318440.390148:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 368 00000100:00080000:2.0:1767318440.390206:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318440.390210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318440.396044:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982562304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5960us (6265us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318440.396061:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 369 00000100:00100000:0.0:1767318440.397354:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982562304:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318440.397377:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982562176:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318442.052536:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318443.973030:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318443.973041:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318443.973907:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318443.973914:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318443.973917:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318443.973930:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3687 00000100:00080000:2.0:1767318443.973934:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318443.973941:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318443.973970:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982562560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318443.974056:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982562560:0@lo:38:kworker.0 00000100:00100000:1.0:1767318443.974131:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318443.974381:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562560 00000100:00100000:0.0:1767318443.974389:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562560 00000100:00100000:0.0:1767318443.974500:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 670 00000100:00100000:0.0:1767318443.974508:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982562560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318443.974531:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318443.974584:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982562560:12345-0@lo:38:kworker.0 Request processed in 75us (453us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318443.974592:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 670 00000100:00080000:1.0:1767318443.974919:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318443.974925:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318443.974928:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982562560:0@lo:38:kworker.0 00000100:00080000:2.0:1767318443.975715:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318443.975721:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318443.975723:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318443.975728:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3687 00000100:00080000:2.0:1767318443.975741:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318443.975747:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318443.975768:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163982562816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318443.975784:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318443.975787:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318443.975789:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318443.975792:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3687 00000100:00080000:2.0:1767318443.975794:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318443.975797:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318443.975809:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198bb80 x1853163982562944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318443.975843:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318443.975846:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318443.975848:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318443.975851:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3687 00000100:00080000:2.0:1767318443.975854:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318443.975857:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318443.975868:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7480 x1853163982563328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318443.975898:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982563200:0@lo:400:kworker.0 00000100:00100000:2.0:1767318443.975932:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318443.975958:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318443.975964:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982563072:0@lo:400:kworker.0 00000100:00100000:2.0:1767318443.975987:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318443.976012:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982562816:0@lo:38:kworker.0 00000100:00100000:2.0:1767318443.976033:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318443.976047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982562944:0@lo:38:kworker.0 00000100:00100000:2.0:1767318443.976072:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318443.976088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982563328:0@lo:38:kworker.0 00000100:00100000:2.0:1767318443.976108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318443.976133:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562944 00000100:00100000:2.0:1767318443.976296:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562944 00000100:00100000:2.0:1767318443.976321:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 193 00000100:00100000:2.0:1767318443.976326:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982562944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318443.976476:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318443.976513:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982562944:12345-0@lo:38:kworker.0 Request processed in 185us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318443.976520:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 193 00000100:00080000:2.0:1767318443.976548:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318443.976552:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318443.976555:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982562944:0@lo:38:kworker.0 00000100:00100000:1.0:1767318443.976651:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982562816 00000100:00100000:1.0:1767318443.976655:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982562816 00000100:00100000:1.0:1767318443.976672:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982563328 00000100:00100000:1.0:1767318443.976674:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982563328 00000100:00100000:1.0:1767318443.976684:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00100000:1.0:1767318443.976688:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982562816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318443.976699:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318443.976804:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982562816:12345-0@lo:38:kworker.0 Request processed in 114us (770us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318443.976812:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00100000:1.0:1767318443.976827:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00100000:1.0:1767318443.976831:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982563328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318443.976840:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318443.976864:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982563328:12345-0@lo:38:kworker.0 Request processed in 32us (756us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318443.976870:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00100000:1.0:1767318443.976889:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982563200 00000100:00100000:1.0:1767318443.976893:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982563200 00000100:00100000:1.0:1767318443.976931:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982563072 00000100:00100000:1.0:1767318443.976935:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982563072 00000100:00100000:1.0:1767318443.976958:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 349 00000100:00100000:1.0:1767318443.976964:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982563200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318443.977001:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982563200:12345-0@lo:400:kworker.0 Request processed in 38us (1068us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318443.977007:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 349 00000100:00100000:1.0:1767318443.977019:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 350 00000100:00100000:1.0:1767318443.977025:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345147f4450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982563072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318443.977051:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345147f4450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982563072:12345-0@lo:400:kworker.0 Request processed in 26us (1063us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318443.977057:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 350 00000100:00080000:2.0:1767318443.978115:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318443.978123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318443.978127:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982562816:0@lo:38:kworker.0 00000100:00080000:2.0:1767318443.978267:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318443.978292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318443.978296:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982563328:0@lo:38:kworker.0 00000100:00100000:0.0:1767318444.940953:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915568896 00000100:00100000:0.0:1767318444.940964:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915568896 00000100:00100000:0.0:1767318444.941009:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569024 00000100:00100000:0.0:1767318444.941013:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569024 00000100:00100000:0.0:1767318444.941073:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 520 00000100:00080000:1.0:1767318444.941084:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318444.941085:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915569024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318444.941184:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915569024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (512us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318444.941193:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 520 00000100:00100000:0.0:1767318444.941211:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 673 00000100:00100000:0.0:1767318444.941215:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915568896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318444.941233:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318444.941261:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915568896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (1108us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318444.941267:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 673 00010000:00010000:1.0:1767318445.125703:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318445.508595:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982563456:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318445.508661:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318445.508709:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318445.508715:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982563584:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318445.508737:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318445.514539:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 370 00000100:00100000:2.0:1767318445.514551:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982563456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318445.514584:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318445.514669:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982563456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (6007us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318445.514677:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 370 00000100:00100000:2.0:1767318445.514733:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982563456:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318445.516940:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318445.516949:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318445.517406:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982563584:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318448.197292:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318449.097395:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318449.097406:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318449.097482:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318449.097486:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318449.097489:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318449.097497:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3692 00000100:00080000:1.0:1767318449.097501:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318449.097507:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318449.097528:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580a80 x1853163982563840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318449.097533:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982563712:0@lo:400:kworker.0 00000100:00080000:1.0:1767318449.097552:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318449.097555:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318449.097556:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318449.097560:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3692 00000100:00080000:1.0:1767318449.097562:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318449.097565:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318449.097573:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582300 x1853163982564096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318449.097580:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318449.097583:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318449.097584:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318449.097587:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3692 00000100:00080000:1.0:1767318449.097589:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318449.097592:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318449.097598:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318449.097601:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581f80 x1853163982564224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318449.097629:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318449.097631:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318449.097633:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318449.097635:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3692 00000100:00080000:1.0:1767318449.097637:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767318449.097640:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767318449.097640:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318449.097645:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982563968:0@lo:400:kworker.0 00000100:00100000:1.0:1767318449.097648:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581180 x1853163982564608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318449.097687:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318449.097695:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982563712 00000100:00100000:1.0:1767318449.097702:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982563712 00000100:00100000:0.0:1767318449.097710:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982564480:0@lo:400:kworker.0 00000100:00100000:0.0:1767318449.097733:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318449.097748:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982563968 00000100:00100000:0.0:1767318449.097750:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318449.097751:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982563968 00000100:00100000:0.0:1767318449.097754:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982564352:0@lo:400:kworker.0 00000100:00100000:0.0:1767318449.097772:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318449.097774:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 521 00000100:00100000:1.0:1767318449.097781:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982563712:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318449.097791:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318449.097803:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 522 00000100:00100000:0.0:1767318449.097809:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982563968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318449.097835:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982563712:12345-0@lo:400:kworker.0 Request processed in 54us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318449.097842:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 521 00010000:00080000:0.0:1767318449.097853:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a580e00 x1853163982563712/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318465 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318449.097863:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982564480 00000100:00100000:0.0:1767318449.097865:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982563712:0@lo:400:kworker.0 00000100:00100000:1.0:1767318449.097866:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982564480 00000100:00080000:0.0:1767318449.097881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318449.097889:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982564352 00000100:00100000:1.0:1767318449.097892:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982564352 00000100:00100000:0.0:1767318449.097917:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982563968:12345-0@lo:400:kworker.0 Request processed in 108us (229us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318449.097917:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 351 00000100:00100000:1.0:1767318449.097923:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982564480:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318449.097924:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 522 00000100:00080000:0.0:1767318449.097936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:0.0:1767318449.097951:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a582680 x1853163982563968/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318465 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318449.097954:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982564480:12345-0@lo:400:kworker.0 Request processed in 31us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318449.097960:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 351 00000100:00100000:0.0:1767318449.097961:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982563968:0@lo:400:kworker.0 00000100:00100000:1.0:1767318449.097970:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 352 00000100:00100000:1.0:1767318449.097976:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982564352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318449.097979:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982564480:0@lo:400:kworker.0 00000100:00100000:1.0:1767318449.098012:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982564352:12345-0@lo:400:kworker.0 Request processed in 36us (239us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318449.098018:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 352 00000100:00100000:0.0:1767318449.098020:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982564352:0@lo:400:kworker.0 00000100:00100000:0.0:1767318449.098051:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982563840:0@lo:38:kworker.0 00000100:00100000:0.0:1767318449.098075:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318449.098090:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982564096:0@lo:38:kworker.0 00000100:00100000:0.0:1767318449.098106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318449.098123:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982564224:0@lo:38:kworker.0 00000100:00100000:0.0:1767318449.098154:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318449.098184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982564608:0@lo:38:kworker.0 00000100:00100000:0.0:1767318449.098203:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318449.107993:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982564608 00000100:00100000:2.0:1767318449.108007:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982564608 00000100:00100000:2.0:1767318449.108059:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 676 00000100:00100000:2.0:1767318449.108068:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982564608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318449.108089:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318449.108222:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318449.108228:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318449.108232:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982563840:0@lo:38:kworker.0 00000100:00100000:2.0:1767318449.111200:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982564608:12345-0@lo:38:kworker.0 Request processed in 3124us (12986us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318449.111216:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 676 00000100:00080000:0.0:1767318449.112802:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318449.112810:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318449.112815:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982564096:0@lo:38:kworker.0 00000100:00080000:0.0:1767318449.113238:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318449.113243:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318449.113246:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982564224:0@lo:38:kworker.0 00000100:00080000:0.0:1767318449.113257:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318449.113260:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318449.113263:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982564608:0@lo:38:kworker.0 00000100:00100000:1.0:1767318450.060829:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318450.060875:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318450.060952:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569280 00000100:00100000:2.0:1767318450.060962:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569280 00000100:00100000:2.0:1767318450.061042:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569152 00000100:00100000:2.0:1767318450.061046:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569152 00000100:00100000:2.0:1767318450.061068:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 523 00000100:00100000:2.0:1767318450.061078:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915569280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318450.061147:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915569280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318450.061154:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 523 00000100:00100000:2.0:1767318450.061170:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 677 00000100:00100000:2.0:1767318450.061175:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915569152:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318450.061195:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318450.061217:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915569152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (342us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318450.061224:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 677 00000100:00080000:0.0:1767318450.061417:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318450.629788:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318450.629810:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982564736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318450.629875:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318450.630098:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318450.630110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982564864:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318450.630150:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318450.637974:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982564736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318450.642169:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982564864:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767318451.269129:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318454.212520:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318454.212535:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318454.212602:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318454.212606:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318454.212610:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767318454.212733:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318454.212750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982564992:0@lo:400:kworker.0 00000100:00100000:0.0:1767318454.212823:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318454.213465:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318454.213520:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3697 00000100:00080000:1.0:1767318454.213526:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318454.213531:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318454.213558:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580700 x1853163982565120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318454.213818:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982565120:0@lo:38:kworker.0 00000100:00100000:0.0:1767318454.213847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318454.214012:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565248 00000100:00100000:2.0:1767318454.214019:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565248 00000100:00080000:2.0:1767318454.214042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318454.214072:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565120 00000100:00100000:2.0:1767318454.214075:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565120 00000100:00100000:2.0:1767318454.214092:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 525 00000100:00100000:2.0:1767318454.214100:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982565248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318454.214191:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 678 00000100:00100000:0.0:1767318454.214197:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565120:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318454.214228:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318454.214264:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565120:12345-0@lo:38:kworker.0 Request processed in 66us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318454.214289:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 678 00000100:00080000:0.0:1767318454.214320:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318454.214326:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318454.214329:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982565120:0@lo:38:kworker.0 00000100:00080000:1.0:1767318454.215682:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318454.215692:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318454.215696:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318454.215702:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3697 00000100:00080000:1.0:1767318454.215706:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318454.215713:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318454.215750:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582680 x1853163982565376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318454.215775:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318454.215778:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318454.215780:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318454.215783:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3697 00000100:00080000:1.0:1767318454.215786:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318454.215789:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318454.215802:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982565504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318454.215862:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318454.215866:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318454.215868:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318454.215871:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3697 00000100:00080000:1.0:1767318454.215874:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318454.215877:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318454.215889:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582300 x1853163982565888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318454.215926:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982565632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.215982:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318454.216022:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318454.216028:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982565760:0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.216047:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318454.216087:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565632 00000100:00100000:1.0:1767318454.216093:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565632 00000100:00100000:1.0:1767318454.216141:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565760 00000100:00100000:1.0:1767318454.216144:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565760 00000100:00100000:1.0:1767318454.216170:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 353 00000100:00100000:1.0:1767318454.216178:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982565632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.216241:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982565632:12345-0@lo:400:kworker.0 Request processed in 64us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318454.216249:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 353 00000100:00100000:1.0:1767318454.216268:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 354 00000100:00100000:1.0:1767318454.216302:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982565760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.216331:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982565760:12345-0@lo:400:kworker.0 Request processed in 30us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318454.216338:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 354 00000100:00080000:1.0:1767318454.216362:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318454.216366:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318454.216391:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982565760:0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.216402:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982565632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318454.216425:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982565376:0@lo:38:kworker.0 00000100:00100000:1.0:1767318454.216451:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318454.216480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982565504:0@lo:38:kworker.0 00000100:00100000:1.0:1767318454.216500:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318454.216522:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982565888:0@lo:38:kworker.0 00000100:00100000:1.0:1767318454.216542:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318454.216569:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565504 00000100:00100000:1.0:1767318454.216573:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565504 00000100:00100000:1.0:1767318454.216599:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 195 00000100:00100000:1.0:1767318454.216603:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982565504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318454.216622:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318454.216656:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982565504:12345-0@lo:38:kworker.0 Request processed in 51us (154us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318454.216662:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 195 00000100:00100000:1.0:1767318454.216679:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565376 00000100:00100000:1.0:1767318454.216683:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565376 00000100:00100000:1.0:1767318454.216695:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982565888 00000100:00100000:1.0:1767318454.216698:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982565888 00000100:00100000:1.0:1767318454.216707:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 679 00000100:00100000:1.0:1767318454.216712:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318454.216722:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318454.216747:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565376:12345-0@lo:38:kworker.0 Request processed in 33us (295us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318454.216752:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 679 00000100:00100000:1.0:1767318454.216765:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 680 00000100:00100000:1.0:1767318454.216769:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318454.216777:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318454.216799:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982565888:12345-0@lo:38:kworker.0 Request processed in 29us (255us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318454.216804:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 680 00000100:00080000:1.0:1767318454.216828:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318454.216833:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318454.216836:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982565376:0@lo:38:kworker.0 00000100:00080000:1.0:1767318454.216844:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318454.216847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318454.216849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982565504:0@lo:38:kworker.0 00000100:00080000:1.0:1767318454.216873:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318454.216877:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318454.216879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982565888:0@lo:38:kworker.0 00000100:00100000:2.0:1767318454.222314:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982565248:12345-0@lo:400:kworker.0 Request processed in 8204us (8628us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318454.222331:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 525 00010000:00010000:2.0:1767318454.340548:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318455.181842:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569536 00000100:00100000:2.0:1767318455.181852:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569536 00000100:00100000:2.0:1767318455.181928:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569408 00000100:00100000:2.0:1767318455.181932:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569408 00000100:00100000:2.0:1767318455.181951:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 526 00000100:00100000:2.0:1767318455.181961:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915569536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318455.182036:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915569536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318455.182044:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 526 00000100:00100000:2.0:1767318455.182165:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 681 00000100:00100000:2.0:1767318455.182170:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915569408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318455.182200:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318455.182232:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915569408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (482us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318455.182238:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 681 00000100:00100000:2.0:1767318455.751819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982566016:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318455.751895:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318455.751935:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566016 00000100:00100000:2.0:1767318455.751944:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566016 00000100:00100000:2.0:1767318455.751991:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 374 00000100:00100000:2.0:1767318455.751999:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982566016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318455.752029:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318455.752082:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982566016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318455.752090:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 374 00000100:00100000:2.0:1767318455.752166:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982566016:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318455.753529:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318455.753549:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982566144:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318455.753611:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318455.753771:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566144 00000100:00100000:2.0:1767318455.753779:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566144 00000100:00100000:2.0:1767318455.753822:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 375 00000100:00100000:2.0:1767318455.753831:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982566144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318455.753855:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318455.753903:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982566144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318455.753911:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 375 00000100:00100000:0.0:1767318455.754108:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982566144:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767318457.412720:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318459.332742:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318459.332753:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318459.332844:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318459.332849:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318459.332852:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318459.332860:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3702 00000100:00080000:0.0:1767318459.332865:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318459.332872:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318459.332895:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163982566400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318459.332933:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318459.332936:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318459.332938:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318459.332953:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3702 00000100:00080000:0.0:1767318459.332955:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318459.332959:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318459.332971:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982566656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318459.332979:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318459.332982:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318459.332983:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318459.332986:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3702 00000100:00080000:0.0:1767318459.332988:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318459.332991:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318459.332999:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa300 x1853163982566784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318459.333037:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318459.333046:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318459.333048:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318459.333051:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3702 00000100:00080000:0.0:1767318459.333053:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318459.333056:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318459.333066:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982567168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318459.333089:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982566912:0@lo:400:kworker.0 00000100:00100000:0.0:1767318459.333197:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318459.333460:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982566272:0@lo:400:kworker.0 00000100:00100000:1.0:1767318459.333512:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318459.333578:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566912 00000100:00100000:1.0:1767318459.333586:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566912 00000100:00100000:1.0:1767318459.333643:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 355 00000100:00100000:1.0:1767318459.333652:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982566912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318459.333702:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982566912:12345-0@lo:400:kworker.0 Request processed in 50us (505us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318459.333710:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 355 00000100:00100000:1.0:1767318459.333749:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566272 00000100:00100000:1.0:1767318459.333753:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566272 00000100:00100000:1.0:1767318459.333784:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 527 00000100:00100000:1.0:1767318459.333792:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982566272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318459.333828:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982566272:12345-0@lo:400:kworker.0 Request processed in 37us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318459.333834:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 527 00000100:00100000:0.0:1767318459.333977:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982566912:0@lo:400:kworker.0 00000100:00100000:0.0:1767318459.333995:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318459.334000:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982567040:0@lo:400:kworker.0 00000100:00100000:0.0:1767318459.334026:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318459.334145:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982566528:0@lo:400:kworker.0 00000100:00100000:2.0:1767318459.334176:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318459.334515:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982567040 00000100:00100000:1.0:1767318459.334521:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982567040 00000100:00100000:1.0:1767318459.334566:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 356 00000100:00100000:1.0:1767318459.334572:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982567040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318459.334612:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982567040:12345-0@lo:400:kworker.0 Request processed in 40us (585us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318459.334617:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 356 00010000:00080000:1.0:1767318459.334648:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ea680 x1853163982566272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318475 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318459.334659:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982566272:0@lo:400:kworker.0 00000100:00100000:1.0:1767318459.334687:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982567040:0@lo:400:kworker.0 00000100:00100000:2.0:1767318459.335446:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566528 00000100:00100000:2.0:1767318459.335452:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566528 00000100:00100000:2.0:1767318459.335491:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566400 00000100:00100000:2.0:1767318459.335495:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566400 00000100:00100000:2.0:1767318459.335509:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566656 00000100:00100000:2.0:1767318459.335511:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566656 00000100:00100000:2.0:1767318459.335523:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 528 00000100:00100000:2.0:1767318459.335541:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982566528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318459.335587:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982566528:12345-0@lo:400:kworker.0 Request processed in 58us (1410us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318459.335594:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 528 00000100:00100000:2.0:1767318459.335609:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 682 00000100:00100000:2.0:1767318459.335613:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982566400:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318459.335631:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318459.335659:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982566400:12345-0@lo:38:kworker.0 Request processed in 44us (569us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318459.335665:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 682 00000100:00100000:2.0:1767318459.335676:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00100000:2.0:1767318459.335679:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982566656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318459.335688:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318459.335709:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982566656:12345-0@lo:38:kworker.0 Request processed in 28us (543us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318459.335713:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00010000:00080000:2.0:1767318459.335749:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9880 x1853163982566528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318475 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318459.335760:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982566528:0@lo:400:kworker.0 00000100:00080000:2.0:1767318459.335798:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318459.336563:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982566784 00000100:00100000:2.0:1767318459.336569:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982566784 00000100:00100000:2.0:1767318459.336599:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 196 00000100:00100000:2.0:1767318459.336603:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982566784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318459.336615:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318459.336656:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982566784:12345-0@lo:38:kworker.0 Request processed in 51us (1404us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318459.336662:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 196 00000100:00080000:0.0:1767318459.336726:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318459.336730:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318459.336734:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982566784:0@lo:38:kworker.0 00000100:00100000:1.0:1767318460.310515:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915569920 00000100:00100000:1.0:1767318460.310525:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915569920 00000100:00100000:1.0:1767318460.310622:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 529 00000100:00100000:1.0:1767318460.310633:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915569920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318460.310708:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915569920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (322us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318460.310715:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 529 00000100:00080000:2.0:1767318460.310792:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767318460.313884:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915569664 offset 0 length 520: 4 00000100:00100000:2.0:1767318460.313934:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00010000:00010000:2.0:1767318460.484481:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318460.868939:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982567296:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318460.869031:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318460.869251:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982567296 00000100:00100000:2.0:1767318460.869260:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982567296 00000100:00100000:2.0:1767318460.869334:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 376 00000100:00100000:2.0:1767318460.869347:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982567296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318460.869394:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318460.884337:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982567296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 14982us (15293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318460.884355:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 376 00000100:00100000:0.0:1767318460.884708:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982567296:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318463.428407:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318438/real 1767318438] req@ffff93463396f800 x1853163982561024/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318463 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318463.428446:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463396f800 x1853163982561024/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318463 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318463.428473:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318463.428501:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318463.428506:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982561024:0@lo:250:kworker.0 00000100:00080000:2.0:1767318463.428594:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318463.428599:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318463.428602:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318463.428606:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3687 00000100:00080000:2.0:1767318463.428610:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318463.428617:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318463.428650:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37100 x1853163982567552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318463.428782:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982567680:0@lo:400:kworker.0 00000100:00100000:1.0:1767318463.428884:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318463.428975:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318463.428991:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318463.428993:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318463.428997:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3707 00000100:00080000:2.0:1767318463.429000:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318463.429003:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318463.429021:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163982567808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318463.429298:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982567552:0@lo:250:kworker.0 00000400:00000100:0.0:1767318463.429321:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982567552 offset 0 length 520: 4 00000100:00100000:0.0:1767318463.429340:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982567808:0@lo:38:kworker.0 00000100:00100000:0.0:1767318463.429359:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318463.429397:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318463.429408:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982567936:0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.429430:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.429465:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982567808 00000100:00100000:0.0:1767318463.429470:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982567808 00000100:00100000:0.0:1767318463.429488:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982567936 00000100:00100000:0.0:1767318463.429491:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982567936 00000100:00100000:0.0:1767318463.429514:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 531 00000100:00100000:0.0:1767318463.429520:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982567936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.429555:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982567936:12345-0@lo:400:kworker.0 Request processed in 34us (123us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318463.429562:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 531 00000100:00100000:0.0:1767318463.429579:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00100000:0.0:1767318463.429583:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982567808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318463.429617:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318463.429646:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982567808:12345-0@lo:38:kworker.0 Request processed in 62us (286us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318463.429655:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00080000:0.0:1767318463.429671:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318463.429691:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee7480 x1853163982567936/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318479 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318463.429702:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982567936:0@lo:400:kworker.0 00000100:00080000:0.0:1767318463.429724:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318463.429727:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318463.429730:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982567808:0@lo:38:kworker.0 00000100:00080000:2.0:1767318463.433156:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318463.433168:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318463.433172:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318463.433180:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3707 00000100:00080000:2.0:1767318463.433184:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318463.433191:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318463.433232:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163982568064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318463.433252:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318463.433255:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318463.433257:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318463.433261:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3707 00000100:00080000:2.0:1767318463.433263:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318463.433266:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318463.433298:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7480 x1853163982568192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318463.433333:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318463.433337:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318463.433339:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318463.433342:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3707 00000100:00080000:2.0:1767318463.433344:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318463.433348:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318463.433357:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b100 x1853163982568576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318463.435079:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982568320:0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.435158:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.435220:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318463.435227:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982568448:0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.435250:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.435318:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982568064:0@lo:38:kworker.0 00000100:00100000:0.0:1767318463.435344:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.435362:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982568192:0@lo:38:kworker.0 00000100:00100000:0.0:1767318463.435385:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.435407:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982568576:0@lo:38:kworker.0 00000100:00100000:0.0:1767318463.435425:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318463.435454:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982568064 00000100:00100000:0.0:1767318463.435461:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982568064 00000100:00100000:0.0:1767318463.435492:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982568576 00000100:00100000:0.0:1767318463.435495:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982568576 00000100:00100000:0.0:1767318463.435513:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 687 00000100:00100000:0.0:1767318463.435520:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982568064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318463.435540:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318463.435582:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982568064:12345-0@lo:38:kworker.0 Request processed in 61us (236us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318463.435589:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 687 00000100:00100000:0.0:1767318463.435605:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 688 00000100:00100000:0.0:1767318463.435609:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982568576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318463.435618:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318463.435641:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982568576:12345-0@lo:38:kworker.0 Request processed in 31us (215us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318463.435646:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 688 00000100:00080000:0.0:1767318463.435685:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318463.435689:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318463.435693:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982568064:0@lo:38:kworker.0 00000100:00080000:0.0:1767318463.435701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318463.435704:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318463.435707:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982568576:0@lo:38:kworker.0 00000100:00100000:0.0:1767318463.435746:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982568320 00000100:00100000:0.0:1767318463.435750:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982568320 00000100:00100000:0.0:1767318463.435787:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982568448 00000100:00100000:0.0:1767318463.435790:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982568448 00000100:00100000:0.0:1767318463.436379:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 357 00000100:00100000:0.0:1767318463.436390:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982568320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.436440:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982568320:12345-0@lo:400:kworker.0 Request processed in 53us (1281us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318463.436446:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 357 00000100:00100000:0.0:1767318463.436459:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 358 00000100:00100000:0.0:1767318463.436466:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982568448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.436493:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982568448:12345-0@lo:400:kworker.0 Request processed in 27us (1242us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318463.436498:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 358 00000100:00080000:0.0:1767318463.436516:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318463.436520:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318463.436541:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982568448:0@lo:400:kworker.0 00000100:00100000:0.0:1767318463.436552:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982568320:0@lo:400:kworker.0 00000100:00080000:0.0:1767318463.438890:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318463.438901:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318463.438905:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982568192:0@lo:38:kworker.0 00010000:00080000:1.0:1767318463.439283:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34a80 x1853163982567680/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318479 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318463.439297:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982567680:0@lo:400:kworker.0 00010000:00010000:2.0:1767318463.559011:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318465.421415:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318465.421506:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318465.421624:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915570048 00000100:00100000:0.0:1767318465.421634:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915570048 00000100:00100000:0.0:1767318465.421700:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 532 00000100:00100000:0.0:1767318465.421712:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915570176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318465.421927:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318465.425369:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915570176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 3645us (3951us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318465.425385:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 532 00000100:00100000:0.0:1767318465.988639:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982568704:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318465.988714:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318465.989219:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318465.989229:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318465.989239:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982568832:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318465.989268:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318465.998846:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318466.630775:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318468.548404:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318468.548414:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318468.548493:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318468.548498:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318468.548502:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318468.548510:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3711 00000100:00080000:2.0:1767318468.548515:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318468.548522:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318468.548544:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163982569088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318468.548580:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318468.548583:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318468.548585:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318468.548588:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3711 00000100:00080000:2.0:1767318468.548591:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318468.548595:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318468.548605:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b100 x1853163982569344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318468.548613:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318468.548616:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318468.548617:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318468.548620:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3711 00000100:00080000:2.0:1767318468.548622:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318468.548625:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318468.548637:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f5180 x1853163982569472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318468.548666:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318468.548669:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318468.548671:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318468.548674:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3711 00000100:00080000:2.0:1767318468.548676:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318468.548680:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318468.548688:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f5f80 x1853163982569856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318468.548717:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982569728:0@lo:400:kworker.0 00000100:00100000:2.0:1767318468.548783:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318468.548821:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318468.548826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982569600:0@lo:400:kworker.0 00000100:00100000:2.0:1767318468.548844:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318468.548922:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982569728 00000100:00100000:2.0:1767318468.548941:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982569728 00000100:00100000:2.0:1767318468.548988:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982569600 00000100:00100000:2.0:1767318468.548991:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982569600 00000100:00100000:2.0:1767318468.549010:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 359 00000100:00100000:2.0:1767318468.549017:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982569728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318468.549094:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982569728:12345-0@lo:400:kworker.0 Request processed in 76us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318468.549111:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 359 00000100:00100000:2.0:1767318468.549127:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 360 00000100:00100000:2.0:1767318468.549134:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982569600:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318468.549162:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982569600:12345-0@lo:400:kworker.0 Request processed in 29us (316us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318468.549168:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 360 00000100:00100000:2.0:1767318468.549203:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982569600:0@lo:400:kworker.0 00000100:00100000:2.0:1767318468.549213:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982569728:0@lo:400:kworker.0 00000100:00100000:1.0:1767318468.549838:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982569088:0@lo:38:kworker.0 00000100:00100000:1.0:1767318468.549868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318468.549894:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982569344:0@lo:38:kworker.0 00000100:00100000:1.0:1767318468.549912:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318468.549924:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982569472:0@lo:38:kworker.0 00000100:00100000:1.0:1767318468.549960:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318468.549974:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982569856:0@lo:38:kworker.0 00000100:00100000:1.0:1767318468.549991:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318468.550416:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318468.550421:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318468.550425:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982569088:0@lo:38:kworker.0 00000100:00080000:1.0:1767318468.550432:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318468.550435:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318468.550437:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982569344:0@lo:38:kworker.0 00000100:00080000:1.0:1767318468.550444:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318468.550447:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318468.550450:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982569856:0@lo:38:kworker.0 00000100:00080000:0.0:1767318468.550684:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318468.550689:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318468.550692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318468.550709:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982569472 00000100:00100000:0.0:1767318468.550713:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982569472 00000100:00100000:0.0:1767318468.550744:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 198 00000100:00100000:0.0:1767318468.550748:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345148c9850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982569472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318468.550759:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318468.550798:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345148c9850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982569472:12345-0@lo:38:kworker.0 Request processed in 49us (838us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318468.550804:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 198 00000100:00080000:1.0:1767318468.551058:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318468.551062:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318468.551066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982569472:0@lo:38:kworker.0 00010000:00010000:2.0:1767318469.701861:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318470.544190:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318470.544244:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318470.544317:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915570304 00000100:00100000:1.0:1767318470.544330:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915570304 00000100:00100000:2.0:1767318470.544435:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915570432 00000100:00100000:2.0:1767318470.544440:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915570432 00000100:00100000:0.0:1767318470.544494:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 693 00000100:00100000:0.0:1767318470.544503:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915570304:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767318470.544517:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 535 00010000:00020000:0.0:1767318470.544527:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318470.544528:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915570432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318470.544842:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915570304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 338us (663us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318470.544850:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 693 00000100:00100000:2.0:1767318470.552053:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915570432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 7514us (7793us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318470.552068:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 535 00000100:00100000:2.0:1767318471.108668:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318471.108690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982569984:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318471.108762:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318471.108791:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982569984 00000100:00100000:2.0:1767318471.108800:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982570112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318471.108801:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982569984 00000100:00100000:2.0:1767318471.108826:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318471.108923:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 380 00000100:00100000:2.0:1767318471.108935:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982569984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318471.108971:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318471.109060:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982569984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 127us (297us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318471.109069:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 380 00000100:00100000:2.0:1767318471.109163:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982570112:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318471.109176:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982569984:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318471.109210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318471.109216:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767318472.772456:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318473.668434:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318473.668446:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318473.668510:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318473.668515:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318473.668519:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318473.668526:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3717 00000100:00080000:2.0:1767318473.668531:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318473.668538:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318473.668618:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f7480 x1853163982570368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318473.668642:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318473.668645:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318473.668647:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318473.668650:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3717 00000100:00080000:2.0:1767318473.668652:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318473.668668:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318473.668679:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f7b80 x1853163982570624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318473.668688:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318473.668691:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318473.668693:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318473.668696:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3717 00000100:00080000:2.0:1767318473.668698:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318473.668701:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318473.668710:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f5500 x1853163982570752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318473.668737:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318473.668740:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318473.668742:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318473.668745:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3717 00000100:00080000:2.0:1767318473.668747:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318473.668751:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318473.668759:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f6300 x1853163982571136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318473.668795:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982570496:0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.668889:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318473.668957:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318473.668962:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982570240:0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.668981:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318473.669008:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982571008:0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.669034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318473.669053:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318473.669057:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982570880:0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.669075:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318473.669102:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982570496 00000100:00100000:2.0:1767318473.669109:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982570496 00000100:00100000:2.0:1767318473.669154:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982570240 00000100:00100000:2.0:1767318473.669157:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982570240 00000100:00100000:2.0:1767318473.669201:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 536 00000100:00100000:2.0:1767318473.669209:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982570496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.669256:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982570496:12345-0@lo:400:kworker.0 Request processed in 47us (366us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318473.669263:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 536 00000100:00100000:2.0:1767318473.669305:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 537 00000100:00100000:2.0:1767318473.669311:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982570240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318473.669336:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982570240:12345-0@lo:400:kworker.0 Request processed in 26us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318473.669340:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 537 00000100:00080000:2.0:1767318473.669355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318473.669379:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346309f4700 x1853163982570240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318489 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318473.669391:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982570240:0@lo:400:kworker.0 00010000:00080000:2.0:1767318473.669401:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346309f4a80 x1853163982570496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318489 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318473.669408:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982570496:0@lo:400:kworker.0 00000100:00100000:0.0:1767318473.669857:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982571008 00000100:00100000:0.0:1767318473.669863:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982571008 00000100:00100000:0.0:1767318473.669932:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982570880 00000100:00100000:0.0:1767318473.669936:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982570880 00000100:00100000:0.0:1767318473.669966:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 361 00000100:00100000:0.0:1767318473.669975:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982571008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318473.670024:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982571008:12345-0@lo:400:kworker.0 Request processed in 51us (989us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318473.670032:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 361 00000100:00100000:0.0:1767318473.670044:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 362 00000100:00100000:0.0:1767318473.670051:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982570880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318473.670079:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982570880:12345-0@lo:400:kworker.0 Request processed in 29us (1002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318473.670085:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 362 00000100:00100000:0.0:1767318473.670126:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982570880:0@lo:400:kworker.0 00000100:00100000:0.0:1767318473.670136:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982571008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318473.670495:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982570368:0@lo:38:kworker.0 00000100:00100000:1.0:1767318473.670572:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318473.670601:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982570624:0@lo:38:kworker.0 00000100:00100000:0.0:1767318473.670722:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982570368 00000100:00100000:0.0:1767318473.670728:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982570368 00000100:00100000:0.0:1767318473.670760:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 694 00000100:00100000:0.0:1767318473.670766:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982570368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318473.670785:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318473.670821:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982570368:12345-0@lo:38:kworker.0 Request processed in 53us (249us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318473.670838:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 694 00000100:00100000:1.0:1767318473.671022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318473.671048:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982570752:0@lo:38:kworker.0 00000100:00100000:1.0:1767318473.671085:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318473.671129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982571136:0@lo:38:kworker.0 00000100:00100000:1.0:1767318473.671150:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318473.671308:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318473.671314:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318473.671405:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318473.671411:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318473.671414:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982570368:0@lo:38:kworker.0 00000100:00100000:0.0:1767318473.672095:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982570752 00000100:00100000:0.0:1767318473.672103:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982570752 00000100:00100000:0.0:1767318473.672169:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 199 00000100:00100000:0.0:1767318473.672175:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982570752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318473.672192:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318473.672249:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982570752:12345-0@lo:38:kworker.0 Request processed in 72us (1162us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318473.672256:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 199 00000100:00080000:0.0:1767318473.672315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318473.672319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318473.672322:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982570752:0@lo:38:kworker.0 00000100:00100000:0.0:1767318473.675295:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00100000:0.0:1767318473.675302:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982571136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318473.675316:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318473.675373:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982571136:12345-0@lo:38:kworker.0 Request processed in 70us (4221us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318473.675379:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00080000:0.0:1767318473.675460:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318473.675465:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318473.675469:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982571136:0@lo:38:kworker.0 00000100:00080000:0.0:1767318473.676762:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318473.676770:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318473.676774:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982570624:0@lo:38:kworker.0 00000100:00100000:2.0:1767318475.661336:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318475.661387:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:2.0:1767318475.662024:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318475.844562:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318476.228732:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982571392:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318476.228819:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318476.228896:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982571264:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318476.228922:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318476.228984:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982571392 00000100:00100000:2.0:1767318476.228993:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982571392 00000100:00080000:1.0:1767318476.229237:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318476.229243:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318476.229861:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982571264:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318476.231806:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 383 00000100:00100000:2.0:1767318476.231852:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982571392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318476.231896:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318476.231976:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982571392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 135us (3157us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318476.231984:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 383 00000100:00100000:0.0:1767318476.232044:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982571392:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318478.788509:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318478.788521:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318478.788630:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318478.788633:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318478.788637:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318478.788645:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3722 00000100:00080000:0.0:1767318478.788650:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318478.788656:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318478.788676:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015180 x1853163982571648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318478.788713:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318478.788716:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318478.788718:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318478.788721:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3722 00000100:00080000:0.0:1767318478.788723:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318478.788727:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318478.788766:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982571904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318478.788775:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318478.788778:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318478.788780:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318478.788784:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3722 00000100:00080000:0.0:1767318478.788786:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318478.788790:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318478.788803:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982572032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318478.788827:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318478.788830:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318478.788831:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318478.788835:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3722 00000100:00080000:0.0:1767318478.788837:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318478.788841:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318478.788851:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017480 x1853163982572416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318478.788879:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982571776:0@lo:400:kworker.0 00000100:00100000:1.0:1767318478.789131:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982571520:0@lo:400:kworker.0 00000100:00100000:2.0:1767318478.789194:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982571648:0@lo:38:kworker.0 00000100:00100000:1.0:1767318478.794618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318478.796611:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318478.796659:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016680 x1853163982571520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318494 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318478.796673:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982571520:0@lo:400:kworker.0 00000100:00100000:2.0:1767318478.796883:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318478.797039:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982571904:0@lo:38:kworker.0 00000100:00100000:2.0:1767318478.797074:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318478.797313:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982572032:0@lo:38:kworker.0 00000100:00100000:2.0:1767318478.797343:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318478.797366:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982572416:0@lo:38:kworker.0 00000100:00100000:2.0:1767318478.797387:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318478.797486:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572032 00000100:00100000:0.0:1767318478.797492:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572032 00000100:00100000:0.0:1767318478.797518:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 200 00000100:00100000:0.0:1767318478.797524:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982572032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318478.797544:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318478.797578:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982572032:12345-0@lo:38:kworker.0 Request processed in 54us (235us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318478.797585:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 200 00000100:00100000:0.0:1767318478.798413:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767318478.798420:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982571648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318478.798435:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318478.798491:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982571648:12345-0@lo:38:kworker.0 Request processed in 70us (1608us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318478.798498:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767318478.798514:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000100:00100000:0.0:1767318478.798518:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982571904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318478.798527:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318478.798554:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982571904:12345-0@lo:38:kworker.0 Request processed in 34us (1478us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318478.798559:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000100:00080000:2.0:1767318478.798901:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318478.798907:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318478.798911:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982571648:0@lo:38:kworker.0 00000100:00080000:2.0:1767318478.798920:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318478.798924:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318478.798938:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982571904:0@lo:38:kworker.0 00000100:00080000:2.0:1767318478.798949:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318478.798952:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318478.798956:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982572032:0@lo:38:kworker.0 00000100:00100000:0.0:1767318478.799097:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572416 00000100:00100000:0.0:1767318478.799102:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572416 00000100:00100000:0.0:1767318478.799222:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 700 00000100:00100000:0.0:1767318478.799228:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982572416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318478.799240:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318478.799295:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982572416:12345-0@lo:38:kworker.0 Request processed in 66us (1906us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318478.799301:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 700 00000100:00100000:0.0:1767318478.799322:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318478.799345:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982571776 00000100:00100000:0.0:1767318478.799349:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982571776 00000100:00100000:0.0:1767318478.799388:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 540 00000100:00100000:0.0:1767318478.799394:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982571776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318478.799431:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982571776:12345-0@lo:400:kworker.0 Request processed in 38us (109us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318478.799437:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 540 00000100:00080000:2.0:1767318478.800090:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318478.800096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318478.800100:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982572416:0@lo:38:kworker.0 00010000:00080000:0.0:1767318478.800900:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016300 x1853163982571776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318494 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318478.800914:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982571776:0@lo:400:kworker.0 00000100:00080000:0.0:1767318478.801027:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318478.916529:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318480.780794:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318480.780888:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318480.780969:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915570944 00000100:00100000:1.0:1767318480.780980:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915570944 00000100:00100000:1.0:1767318480.781053:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915570816 00000100:00100000:1.0:1767318480.781057:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915570816 00000100:00100000:1.0:1767318480.781080:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 541 00000100:00100000:1.0:1767318480.781090:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915570944:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318480.781165:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915570944:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318480.781173:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 541 00000100:00100000:1.0:1767318480.781192:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 701 00000100:00100000:1.0:1767318480.781197:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915570816:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318480.781215:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318480.781239:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915570816:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318480.781245:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 701 00000100:00080000:0.0:1767318480.781503:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318481.349192:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982572544:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318481.349292:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318481.350879:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318481.350896:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982572672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318481.350959:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318481.351792:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572544 00000100:00100000:2.0:1767318481.351816:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572544 00000100:00100000:2.0:1767318481.351875:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572672 00000100:00100000:2.0:1767318481.351879:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572672 00000100:00100000:2.0:1767318481.351899:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 384 00000100:00100000:1.0:1767318481.351909:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 385 00000100:00100000:2.0:1767318481.351912:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982572544:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318481.351917:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982572672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318481.351943:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:2.0:1767318481.351943:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767318481.352150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318481.352155:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318481.354795:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982572672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2865us (3821us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318481.354812:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 385 00000100:00100000:2.0:1767318481.354923:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982572544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3015us (5650us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318481.354930:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 384 00000100:00100000:1.0:1767318481.357251:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982572544:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767318481.991376:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318483.908473:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318483.908495:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318483.908565:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318483.908570:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318483.908574:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318483.908582:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3727 00000100:00080000:0.0:1767318483.908587:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318483.908593:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318483.908615:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163982572928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318483.908679:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318483.908682:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318483.908684:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318483.908688:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3727 00000100:00080000:0.0:1767318483.908691:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318483.908694:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318483.908706:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982573184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318483.908716:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318483.908720:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318483.908721:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318483.908725:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3727 00000100:00080000:0.0:1767318483.908728:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318483.908731:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318483.908747:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982573312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318483.908770:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318483.908773:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318483.908775:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318483.908778:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3727 00000100:00080000:0.0:1767318483.908781:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318483.908784:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318483.908794:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982573696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318483.908839:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982573056:0@lo:400:kworker.0 00000100:00100000:0.0:1767318483.908905:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318483.908973:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318483.908979:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982573440:0@lo:400:kworker.0 00000100:00100000:0.0:1767318483.909005:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318483.909034:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573056 00000100:00100000:0.0:1767318483.909041:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573056 00000100:00100000:0.0:1767318483.909107:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 542 00000100:00100000:0.0:1767318483.909117:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982573056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318483.909165:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982573056:12345-0@lo:400:kworker.0 Request processed in 51us (261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318483.909173:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 542 00000100:00080000:0.0:1767318483.909194:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318483.909221:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017480 x1853163982573056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318499 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318483.909233:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982573056:0@lo:400:kworker.0 00000100:00100000:2.0:1767318483.909350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982572928:0@lo:38:kworker.0 00000100:00100000:2.0:1767318483.909379:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318483.909409:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982573184:0@lo:38:kworker.0 00000100:00100000:2.0:1767318483.909430:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318483.909443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982573312:0@lo:38:kworker.0 00000100:00100000:2.0:1767318483.909462:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318483.909480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982573696:0@lo:38:kworker.0 00000100:00100000:2.0:1767318483.909497:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318483.909571:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982573568:0@lo:400:kworker.0 00000100:00100000:1.0:1767318483.909599:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318483.909628:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318483.909644:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982572800:0@lo:400:kworker.0 00000100:00100000:1.0:1767318483.909665:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318483.909690:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572928 00000100:00100000:1.0:1767318483.909693:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572928 00000100:00100000:1.0:1767318483.909707:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573184 00000100:00100000:1.0:1767318483.909709:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573184 00000100:00100000:1.0:1767318483.909715:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573696 00000100:00100000:1.0:1767318483.909717:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573696 00000100:00100000:1.0:1767318483.909724:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982572800 00000100:00100000:1.0:1767318483.909726:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982572800 00000100:00100000:1.0:1767318483.909748:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 543 00000100:00100000:1.0:1767318483.909754:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982572800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318483.909786:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982572800:12345-0@lo:400:kworker.0 Request processed in 32us (120us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318483.909792:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 543 00000100:00100000:1.0:1767318483.909804:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767318483.909809:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982572928:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318483.909827:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318483.909857:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982572928:12345-0@lo:38:kworker.0 Request processed in 47us (476us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318483.909865:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767318483.909877:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 703 00000100:00100000:1.0:1767318483.909880:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982573184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318483.909888:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318483.909910:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982573184:12345-0@lo:38:kworker.0 Request processed in 28us (479us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318483.909915:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 703 00000100:00100000:1.0:1767318483.909924:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00100000:1.0:1767318483.909928:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982573696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318483.909935:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318483.909955:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982573696:12345-0@lo:38:kworker.0 Request processed in 25us (456us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318483.909959:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00000100:00080000:1.0:1767318483.909979:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318483.909997:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016300 x1853163982572800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318499 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318483.910007:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982572800:0@lo:400:kworker.0 00000100:00080000:1.0:1767318483.910028:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318483.910032:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318483.910035:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982572928:0@lo:38:kworker.0 00000100:00080000:1.0:1767318483.910044:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318483.910047:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318483.910062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982573184:0@lo:38:kworker.0 00000100:00080000:1.0:1767318483.910074:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318483.910077:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318483.910079:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982573696:0@lo:38:kworker.0 00000100:00100000:2.0:1767318483.910151:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573440 00000100:00100000:2.0:1767318483.910155:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573440 00000100:00100000:2.0:1767318483.910192:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573568 00000100:00100000:2.0:1767318483.910196:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573568 00000100:00100000:2.0:1767318483.910213:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 365 00000100:00100000:2.0:1767318483.910219:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982573440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318483.910254:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982573440:12345-0@lo:400:kworker.0 Request processed in 34us (1248us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318483.910260:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 365 00000100:00100000:2.0:1767318483.910299:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 366 00000100:00100000:2.0:1767318483.910307:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982573568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318483.910340:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982573568:12345-0@lo:400:kworker.0 Request processed in 34us (740us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318483.910346:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 366 00000100:00100000:2.0:1767318483.910373:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982573568:0@lo:400:kworker.0 00000100:00080000:1.0:1767318483.910431:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318483.910434:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318483.911118:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573312 00000100:00100000:0.0:1767318483.911125:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573312 00000100:00100000:0.0:1767318483.911147:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 201 00000100:00100000:0.0:1767318483.911151:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982573312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318483.911166:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318483.911229:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982573312:12345-0@lo:38:kworker.0 Request processed in 76us (1766us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318483.911236:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 201 00000100:00080000:1.0:1767318483.911266:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318483.911288:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318483.911292:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982573312:0@lo:38:kworker.0 00000100:00100000:1.0:1767318483.911332:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982573440:0@lo:400:kworker.0 00000100:00100000:1.0:1767318484.877536:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318484.877682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915571328 00000100:00100000:2.0:1767318484.877692:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915571328 00000100:00100000:2.0:1767318484.877821:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 544 00000100:00100000:2.0:1767318484.877833:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915571328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318484.878203:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915571328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 371us (676us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318484.878212:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 544 00000100:00080000:1.0:1767318484.887621:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767318484.887753:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915571072 offset 0 length 520: 4 00000100:00100000:2.0:1767318484.887805:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318484.888073:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915571200 00000100:00100000:1.0:1767318484.888083:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915571200 00000100:00100000:1.0:1767318484.888136:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 705 00000100:00100000:1.0:1767318484.888146:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915571200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318484.888169:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318484.888244:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915571200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 97us (437us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318484.888251:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 705 00010000:00010000:2.0:1767318485.063733:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318486.468974:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318486.468999:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982573824:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318486.469070:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318486.469937:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318486.470078:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318486.470090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982573952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318486.470127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318486.470209:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982573952 00000100:00100000:0.0:1767318486.470214:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982573952 00000100:00100000:0.0:1767318486.470265:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 387 00000100:00100000:0.0:1767318486.470301:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982573952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318486.470321:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318486.470360:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982573952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (232us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318486.470365:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 387 00000100:00100000:0.0:1767318486.470398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982573952:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318486.470454:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318488.004437:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318463/real 1767318463] req@ffff934609a37100 x1853163982567552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318488 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318488.004475:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609a37100 x1853163982567552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318488 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318488.004509:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318488.004619:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318488.004624:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982567552:0@lo:250:kworker.0 00000100:00080000:0.0:1767318488.007216:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318488.007224:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318488.007228:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318488.007235:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3711 00000100:00080000:0.0:1767318488.007239:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318488.007246:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318488.007297:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc000 x1853163982574080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318488.007382:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318488.007386:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318488.007388:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318488.007391:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3732 00000100:00080000:0.0:1767318488.007394:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318488.007397:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318488.007411:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982574336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318488.007438:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318488.007441:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318488.007443:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318488.007447:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3732 00000100:00080000:0.0:1767318488.007449:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318488.007452:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318488.007460:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982574592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318488.007468:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318488.007470:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318488.007472:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318488.007474:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3732 00000100:00080000:0.0:1767318488.007476:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318488.007479:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318488.007706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982574208:0@lo:400:kworker.0 00000100:00100000:2.0:1767318488.007786:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318488.008122:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318488.008132:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982574464:0@lo:400:kworker.0 00000100:00100000:2.0:1767318488.008161:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318488.009118:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318488.009176:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982574720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318488.009234:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318488.009237:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318488.009240:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318488.009244:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3732 00000100:00080000:0.0:1767318488.009247:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318488.009251:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318488.009263:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017480 x1853163982575104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318488.009346:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318488.009357:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982574848:0@lo:400:kworker.0 00000100:00100000:2.0:1767318488.009394:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318488.009610:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982574848 00000100:00100000:0.0:1767318488.009616:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982574848 00000100:00100000:1.0:1767318488.009796:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982574080:0@lo:250:kworker.0 00000400:00000100:1.0:1767318488.009828:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982574080 offset 0 length 520: 4 00000100:00100000:1.0:1767318488.009857:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982574336:0@lo:38:kworker.0 00000100:00100000:1.0:1767318488.010061:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318488.010087:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982574592:0@lo:38:kworker.0 00000100:00100000:1.0:1767318488.010111:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318488.010123:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982574720:0@lo:38:kworker.0 00000100:00100000:1.0:1767318488.010144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318488.010158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982575104:0@lo:38:kworker.0 00000100:00100000:1.0:1767318488.010173:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318488.010202:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982574336 00000100:00100000:1.0:1767318488.010207:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982574336 00000100:00100000:1.0:1767318488.010227:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982574592 00000100:00100000:1.0:1767318488.010230:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982574592 00000100:00100000:1.0:1767318488.010241:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982575104 00000100:00100000:1.0:1767318488.010244:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982575104 00000100:00100000:1.0:1767318488.010255:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767318488.010260:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982574336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318488.010317:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318488.010446:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 367 00000100:00100000:0.0:1767318488.010456:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982574848:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767318488.010620:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318488.010674:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00100000:2.0:1767318488.010680:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982574592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318488.010690:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318488.010730:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982574592:12345-0@lo:38:kworker.0 Request processed in 48us (617us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318488.010737:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00100000:1.0:1767318488.011024:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982574336:12345-0@lo:38:kworker.0 Request processed in 761us (1142us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318488.011033:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767318488.011049:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00100000:1.0:1767318488.011053:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982575104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318488.011062:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318488.011085:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982575104:12345-0@lo:38:kworker.0 Request processed in 31us (910us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318488.011090:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00080000:1.0:1767318488.011128:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318488.011133:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318488.011136:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982574336:0@lo:38:kworker.0 00000100:00080000:1.0:1767318488.011145:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318488.011148:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318488.011151:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982574592:0@lo:38:kworker.0 00000100:00080000:1.0:1767318488.011159:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318488.011162:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318488.011164:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982575104:0@lo:38:kworker.0 00000100:00100000:1.0:1767318488.011203:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982574720 00000100:00100000:1.0:1767318488.011208:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982574720 00000100:00100000:1.0:1767318488.011231:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 202 00000100:00100000:1.0:1767318488.011236:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982574720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318488.011245:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318488.011306:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982574720:12345-0@lo:38:kworker.0 Request processed in 69us (1160us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318488.011313:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 202 00000100:00100000:0.0:1767318488.011474:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982574848:12345-0@lo:400:kworker.0 Request processed in 1020us (2079us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318488.011482:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 367 00000100:00100000:2.0:1767318488.011631:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982574848:0@lo:400:kworker.0 00000100:00080000:1.0:1767318488.012200:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318488.012206:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318488.012209:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982574720:0@lo:38:kworker.0 00000100:00100000:0.0:1767318488.016579:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982574976 00000100:00100000:0.0:1767318488.016597:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982574976 00000100:00100000:0.0:1767318488.016701:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 368 00000100:00100000:0.0:1767318488.016712:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982574976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318488.016829:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982574976:12345-0@lo:400:kworker.0 Request processed in 118us (4461us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318488.016836:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 368 00000100:00100000:1.0:1767318488.017260:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982574976:0@lo:400:kworker.0 00000100:00080000:2.0:1767318488.017443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767318488.132528:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318490.005687:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318490.005796:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318490.007312:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915571584 00000100:00100000:2.0:1767318490.007326:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915571584 00000100:00100000:2.0:1767318490.007417:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915571456 00000100:00100000:2.0:1767318490.007421:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915571456 00000100:00100000:2.0:1767318490.007448:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 547 00000100:00100000:2.0:1767318490.007458:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915571584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318490.007549:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915571584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (1873us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318490.007557:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 547 00000100:00100000:2.0:1767318490.007576:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 709 00000100:00100000:2.0:1767318490.007581:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915571456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318490.007601:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318490.007629:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915571456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (1833us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318490.007635:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 709 00000100:00080000:1.0:1767318490.007884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318491.204532:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318491.588584:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318491.588608:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982575232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318491.588676:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318491.588956:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982575360:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318491.589028:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318491.589077:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982575232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318491.593157:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318491.593260:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982575360 00000100:00100000:2.0:1767318491.593286:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982575360 00000100:00100000:2.0:1767318491.593347:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 389 00000100:00100000:2.0:1767318491.593357:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982575360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318491.593395:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318491.593478:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982575360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 124us (329us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318491.593495:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 389 00000100:00100000:0.0:1767318491.593639:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982575360:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318491.594033:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318493.126116:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982575488:0@lo:400:kworker.0 00000100:00100000:2.0:1767318493.126166:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318493.126197:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318493.126203:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982576256:0@lo:400:kworker.0 00000100:00100000:2.0:1767318493.126226:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318493.126251:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982576256 00000100:00100000:2.0:1767318493.126255:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982576256 00000100:00100000:2.0:1767318493.126308:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 370 00000100:00100000:2.0:1767318493.126315:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982576256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318493.126352:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982576256:12345-0@lo:400:kworker.0 Request processed in 38us (124us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318493.126359:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 370 00000100:00080000:2.0:1767318493.126386:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318493.126404:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982576256:0@lo:400:kworker.0 00000100:00100000:0.0:1767318493.126683:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982575488 00000100:00100000:0.0:1767318493.126687:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982575488 00000100:00100000:0.0:1767318493.126730:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 549 00000100:00100000:0.0:1767318493.126736:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982575488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318493.126779:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982575488:12345-0@lo:400:kworker.0 Request processed in 43us (613us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318493.126784:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 549 00010000:00080000:0.0:1767318493.126814:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912d80 x1853163982575488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318509 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318493.126824:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982575488:0@lo:400:kworker.0 00000100:00100000:1.0:1767318493.127097:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982575616:0@lo:38:kworker.0 00000100:00100000:1.0:1767318493.127140:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318493.127163:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982575872:0@lo:38:kworker.0 00000100:00100000:1.0:1767318493.127181:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318493.127198:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982576000:0@lo:38:kworker.0 00000100:00100000:1.0:1767318493.127216:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318493.127231:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982576384:0@lo:38:kworker.0 00000100:00100000:1.0:1767318493.127248:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318493.127292:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982576000 00000100:00100000:1.0:1767318493.127296:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982576000 00000100:00100000:1.0:1767318493.127315:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 203 00000100:00100000:1.0:1767318493.127319:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982576000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318493.127337:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318493.127366:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982576000:12345-0@lo:38:kworker.0 Request processed in 46us (149us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318493.127374:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 203 00000100:00080000:1.0:1767318493.127412:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318493.127416:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318493.127419:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982576000:0@lo:38:kworker.0 00000100:00100000:0.0:1767318493.127712:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982575616 00000100:00100000:0.0:1767318493.127716:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982575616 00000100:00100000:0.0:1767318493.127739:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982575872 00000100:00100000:0.0:1767318493.127742:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982575872 00000100:00100000:0.0:1767318493.127753:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982576384 00000100:00100000:0.0:1767318493.127755:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982576384 00000100:00100000:0.0:1767318493.127763:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000100:00100000:0.0:1767318493.127767:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982575616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318493.127778:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318493.127809:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982575616:12345-0@lo:38:kworker.0 Request processed in 41us (680us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318493.127814:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00000100:00100000:0.0:1767318493.127824:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000100:00100000:0.0:1767318493.127839:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982575872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318493.127847:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318493.127866:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982575872:12345-0@lo:38:kworker.0 Request processed in 26us (684us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318493.127871:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00000100:00100000:0.0:1767318493.127880:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 712 00000100:00100000:0.0:1767318493.127883:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982576384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318493.127891:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318493.127915:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982576384:12345-0@lo:38:kworker.0 Request processed in 31us (666us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318493.127920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 712 00000100:00080000:0.0:1767318493.127940:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318493.127943:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318493.127947:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982575616:0@lo:38:kworker.0 00000100:00080000:0.0:1767318493.127953:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318493.127956:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318493.127959:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982575872:0@lo:38:kworker.0 00000100:00080000:0.0:1767318493.127965:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318493.127968:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318493.127971:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982576384:0@lo:38:kworker.0 00000100:00080000:2.0:1767318493.129372:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318493.253631:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318494.276543:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318495.117075:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:2.0:1767318495.117624:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318495.120192:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318495.120372:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915571712 00000100:00100000:1.0:1767318495.120383:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915571712 00000100:00100000:1.0:1767318495.120640:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 713 00000100:00100000:1.0:1767318495.120647:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915571712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318495.120671:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318495.120740:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915571712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 91us (558us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318495.120746:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 713 00010000:00010000:1.0:1767318496.324701:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318496.708824:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982576512:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318496.708903:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318496.709230:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982576512:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318496.709260:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318496.709511:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318496.709520:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982576640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318496.709548:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318496.709611:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982576640 00000100:00100000:2.0:1767318496.709616:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982576640 00000100:00100000:2.0:1767318496.709657:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 391 00000100:00100000:2.0:1767318496.709664:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982576640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318496.709681:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318496.709711:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982576640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318496.709718:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 391 00000100:00100000:1.0:1767318496.709791:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982576640:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318496.709844:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318497.352725:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318498.246646:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982576768:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.246724:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318498.246777:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318498.246782:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982577536:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.246817:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318498.246848:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982577408:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.246872:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318498.246895:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318498.246900:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982577024:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.246930:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318498.246961:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577536 00000100:00100000:1.0:1767318498.246968:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577536 00000100:00100000:1.0:1767318498.247019:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577408 00000100:00100000:1.0:1767318498.247022:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577408 00000100:00100000:1.0:1767318498.247045:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 371 00000100:00100000:1.0:1767318498.247052:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982577536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247101:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982577536:12345-0@lo:400:kworker.0 Request processed in 48us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318498.247109:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 371 00000100:00100000:1.0:1767318498.247126:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 372 00000100:00100000:1.0:1767318498.247133:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982577408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247176:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982577408:12345-0@lo:400:kworker.0 Request processed in 44us (303us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318498.247182:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 372 00000100:00100000:1.0:1767318498.247225:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982577408:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247255:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982577536:0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247300:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982576768 00000100:00100000:1.0:1767318498.247303:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982576768 00000100:00100000:1.0:1767318498.247328:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577024 00000100:00100000:1.0:1767318498.247331:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577024 00000100:00100000:1.0:1767318498.247348:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 551 00000100:00100000:1.0:1767318498.247354:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982576768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247402:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982576768:12345-0@lo:400:kworker.0 Request processed in 48us (677us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318498.247408:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 551 00000100:00100000:1.0:1767318498.247419:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 552 00000100:00100000:1.0:1767318498.247425:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982577024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318498.247452:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982577024:12345-0@lo:400:kworker.0 Request processed in 27us (530us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318498.247457:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 552 00010000:00080000:1.0:1767318498.247485:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910380 x1853163982577024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318514 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318498.247495:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982577024:0@lo:400:kworker.0 00010000:00080000:1.0:1767318498.247516:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910a80 x1853163982576768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318514 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318498.247525:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982576768:0@lo:400:kworker.0 00000100:00080000:0.0:1767318498.248029:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318498.248035:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318498.248038:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318498.249768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982576896:0@lo:38:kworker.0 00000100:00100000:2.0:1767318498.249868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318498.249916:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982577152:0@lo:38:kworker.0 00000100:00100000:2.0:1767318498.249941:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318498.249972:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982577280:0@lo:38:kworker.0 00000100:00100000:2.0:1767318498.250010:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318498.250056:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982577664:0@lo:38:kworker.0 00000100:00100000:2.0:1767318498.250084:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318498.250130:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577152 00000100:00100000:2.0:1767318498.250136:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577152 00000100:00100000:2.0:1767318498.250165:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000100:00100000:2.0:1767318498.250176:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982576896:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767318498.250251:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00100000:0.0:1767318498.250257:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982577152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318498.250317:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318498.250375:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982576896:12345-0@lo:38:kworker.0 Request processed in 200us (507us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318498.250382:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00080000:2.0:1767318498.250421:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318498.250426:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318498.250431:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982576896:0@lo:38:kworker.0 00000100:00100000:1.0:1767318498.250474:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577280 00000100:00100000:1.0:1767318498.250480:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577280 00000100:00100000:1.0:1767318498.250505:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 204 00000100:00100000:1.0:1767318498.250512:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982577280:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767318498.256231:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318498.256239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318498.256243:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982577664:0@lo:38:kworker.0 00010000:00020000:0.0:1767318498.267198:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318498.267328:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982577152:12345-0@lo:38:kworker.0 Request processed in 17066us (17382us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318498.267341:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00010000:00020000:1.0:1767318498.268077:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318498.268133:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982577280:12345-0@lo:38:kworker.0 Request processed in 17619us (18120us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318498.268144:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 204 00000100:00080000:2.0:1767318498.269127:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318498.269135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318498.269140:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982577152:0@lo:38:kworker.0 00000100:00080000:2.0:1767318498.269154:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318498.269158:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318498.269162:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982577280:0@lo:38:kworker.0 00010000:00010000:1.0:1767318499.397050:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318500.242416:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318500.242518:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767318500.243381:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318500.420594:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318501.828624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982577792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318501.828720:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318501.828840:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577792 00000100:00100000:0.0:1767318501.828849:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577792 00000100:00100000:2.0:1767318501.828869:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318501.828882:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982577920:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318501.828913:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318501.828916:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 392 00000100:00100000:0.0:1767318501.828926:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982577792:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318501.828940:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982577920 00000100:00100000:2.0:1767318501.828943:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982577920 00002000:00000024:0.0:1767318501.828956:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318501.828963:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 393 00000100:00100000:2.0:1767318501.828968:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982577920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318501.828983:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318501.829010:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982577920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (96us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318501.829018:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 393 00000100:00080000:2.0:1767318501.829045:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318501.829048:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318501.830305:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982577792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1363us (1567us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318501.830316:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 392 00000100:00100000:1.0:1767318501.830995:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982577792:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318502.472635:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318503.364952:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318503.364962:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318503.365037:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318503.365041:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318503.365045:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318503.365054:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3746 00000100:00080000:1.0:1767318503.365058:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318503.365064:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318503.365086:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900700 x1853163982578176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318503.365133:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318503.365136:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318503.365138:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318503.365142:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3746 00000100:00080000:1.0:1767318503.365145:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318503.365150:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318503.365160:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901880 x1853163982578432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318503.365170:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318503.365173:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318503.365175:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318503.365179:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3746 00000100:00080000:1.0:1767318503.365181:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318503.365185:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318503.365198:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fed80 x1853163982578560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318503.365218:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318503.365222:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318503.365224:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318503.365227:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3746 00000100:00080000:1.0:1767318503.365230:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318503.365233:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318503.365244:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57480 x1853163982578944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318503.365317:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982578688:0@lo:400:kworker.0 00000100:00100000:1.0:1767318503.365402:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318503.365468:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318503.365474:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982578304:0@lo:400:kworker.0 00000100:00100000:1.0:1767318503.365499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318503.365588:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982578176:0@lo:38:kworker.0 00000100:00100000:0.0:1767318503.365618:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318503.365662:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982578432:0@lo:38:kworker.0 00000100:00100000:0.0:1767318503.365684:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318503.365701:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982578560:0@lo:38:kworker.0 00000100:00100000:0.0:1767318503.365724:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318503.365746:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982578944:0@lo:38:kworker.0 00000100:00100000:0.0:1767318503.365764:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318503.365797:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578304 00000100:00100000:0.0:1767318503.365805:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578304 00000100:00100000:0.0:1767318503.365863:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578176 00000100:00100000:0.0:1767318503.365866:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578176 00000100:00100000:0.0:1767318503.365874:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578432 00000100:00100000:0.0:1767318503.365876:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578432 00000100:00100000:0.0:1767318503.365882:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578944 00000100:00100000:0.0:1767318503.365884:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578944 00000100:00100000:0.0:1767318503.365895:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 554 00000100:00100000:0.0:1767318503.365905:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982578304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318503.365961:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982578304:12345-0@lo:400:kworker.0 Request processed in 56us (460us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318503.365969:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 554 00000100:00100000:0.0:1767318503.365986:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 718 00000100:00100000:0.0:1767318503.365990:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318503.366010:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318503.366043:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578176:12345-0@lo:38:kworker.0 Request processed in 52us (424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318503.366049:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 718 00000100:00100000:0.0:1767318503.366061:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00100000:0.0:1767318503.366064:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318503.366073:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318503.366093:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578432:12345-0@lo:38:kworker.0 Request processed in 27us (408us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318503.366098:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00100000:0.0:1767318503.366107:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 00000100:00100000:0.0:1767318503.366110:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318503.366118:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318503.366136:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982578944:12345-0@lo:38:kworker.0 Request processed in 25us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318503.366141:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00000100:00100000:0.0:1767318503.366157:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578560 00000100:00100000:0.0:1767318503.366160:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578560 00000100:00100000:0.0:1767318503.366177:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 205 00000100:00100000:0.0:1767318503.366181:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982578560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318503.366189:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318503.366209:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982578560:12345-0@lo:38:kworker.0 Request processed in 27us (485us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318503.366214:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 205 00010000:00080000:0.0:1767318503.366250:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631902a00 x1853163982578304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318519 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318503.366307:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982578304:0@lo:400:kworker.0 00000100:00080000:0.0:1767318503.366341:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318503.366346:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318503.366349:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982578176:0@lo:38:kworker.0 00000100:00080000:0.0:1767318503.366356:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318503.366359:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318503.366362:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982578432:0@lo:38:kworker.0 00000100:00080000:0.0:1767318503.366368:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318503.366370:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318503.366373:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982578560:0@lo:38:kworker.0 00000100:00080000:0.0:1767318503.366379:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318503.366382:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318503.366384:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982578944:0@lo:38:kworker.0 00000100:00100000:0.0:1767318503.366958:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982578048 00000100:00100000:0.0:1767318503.366962:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982578048 00000100:00100000:0.0:1767318503.366996:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 555 00000100:00100000:0.0:1767318503.367004:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982578048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318503.367039:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982578048:12345-0@lo:400:kworker.0 Request processed in 36us (444us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318503.367044:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 555 00000100:00100000:0.0:1767318503.367073:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982578688:0@lo:400:kworker.0 00000100:00080000:2.0:1767318503.367187:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318503.367191:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318503.367192:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318503.494619:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318505.356315:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:2.0:1767318505.356842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318505.357847:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318505.357938:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915572224 00000100:00100000:0.0:1767318505.357946:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915572224 00000100:00100000:0.0:1767318505.358149:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 721 00000100:00100000:0.0:1767318505.358157:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915572224:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318505.358180:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318505.358245:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915572224:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318505.358252:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 721 00010000:00010000:1.0:1767318505.540526:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318506.564656:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318506.949669:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982579072:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318506.949777:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318506.950912:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982579200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318506.950949:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318506.951605:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982579200:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318506.951874:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318506.951879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318506.952454:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982579072:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318508.486641:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318508.486652:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318508.486757:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982579328:0@lo:400:kworker.0 00000100:00100000:1.0:1767318508.486838:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318508.486936:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318508.486941:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318508.486944:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318508.486949:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3751 00000100:00080000:2.0:1767318508.486953:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318508.486959:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318508.486979:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1c00 x1853163982579456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318508.487008:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318508.487011:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318508.487013:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318508.487016:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3751 00000100:00080000:2.0:1767318508.487018:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318508.487022:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318508.487031:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1f80 x1853163982579712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318508.487038:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318508.487041:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318508.487042:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318508.487045:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3751 00000100:00080000:2.0:1767318508.487047:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318508.487050:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318508.487060:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0e00 x1853163982579840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318508.487319:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318508.487351:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982579456:0@lo:38:kworker.0 00000100:00100000:0.0:1767318508.487385:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.487415:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982579712:0@lo:38:kworker.0 00000100:00100000:0.0:1767318508.487438:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.487457:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982579840:0@lo:38:kworker.0 00000100:00100000:0.0:1767318508.487481:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.487517:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982579456 00000100:00100000:0.0:1767318508.487521:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982579456 00000100:00100000:0.0:1767318508.487542:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982579712 00000100:00100000:0.0:1767318508.487545:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982579712 00000100:00100000:0.0:1767318508.487557:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767318508.487562:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982579456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318508.487580:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318508.487616:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982579456:12345-0@lo:38:kworker.0 Request processed in 52us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318508.487624:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767318508.487640:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00100000:0.0:1767318508.487644:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982579712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318508.487654:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318508.487680:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982579712:12345-0@lo:38:kworker.0 Request processed in 34us (241us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318508.487687:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00000100:00100000:0.0:1767318508.487704:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982579840 00000100:00100000:0.0:1767318508.487708:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982579840 00000100:00100000:0.0:1767318508.487727:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 206 00000100:00100000:0.0:1767318508.487731:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982579840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318508.487741:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318508.487766:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982579840:12345-0@lo:38:kworker.0 Request processed in 33us (284us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318508.487773:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 206 00000100:00080000:0.0:1767318508.487811:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318508.487816:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318508.487820:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982579456:0@lo:38:kworker.0 00000100:00080000:0.0:1767318508.487829:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318508.487832:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318508.487835:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982579712:0@lo:38:kworker.0 00000100:00080000:0.0:1767318508.487843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318508.487847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318508.487849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982579840:0@lo:38:kworker.0 00000100:00100000:0.0:1767318508.487898:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982579584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.487923:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.487943:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318508.487949:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982579968:0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.487973:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.487999:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982580096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488021:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318508.488049:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982579584 00000100:00100000:0.0:1767318508.488053:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982579584 00000100:00100000:0.0:1767318508.488083:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 558 00000100:00100000:0.0:1767318508.488089:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982579584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488124:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982579584:12345-0@lo:400:kworker.0 Request processed in 35us (200us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318508.488130:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 558 00000100:00080000:0.0:1767318508.488146:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318508.488178:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0000 x1853163982579584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318524 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318508.488189:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982579584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488234:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982579968 00000100:00100000:0.0:1767318508.488238:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982579968 00000100:00100000:0.0:1767318508.488301:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580096 00000100:00100000:0.0:1767318508.488304:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580096 00000100:00100000:0.0:1767318508.488327:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 375 00000100:00100000:0.0:1767318508.488334:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982579968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488370:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982579968:12345-0@lo:400:kworker.0 Request processed in 37us (396us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318508.488377:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 375 00000100:00100000:0.0:1767318508.488389:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 376 00000100:00100000:0.0:1767318508.488395:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982580096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488426:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982580096:12345-0@lo:400:kworker.0 Request processed in 31us (404us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318508.488432:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 376 00000100:00080000:0.0:1767318508.488450:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318508.488453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318508.488469:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982580096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318508.488491:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982579968:0@lo:400:kworker.0 00000100:00080000:2.0:1767318508.494301:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318508.494313:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318508.494317:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318508.494326:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3751 00000100:00080000:2.0:1767318508.494331:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318508.494345:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318508.494386:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163982580224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318508.495446:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982580224:0@lo:38:kworker.0 00000100:00100000:0.0:1767318508.495532:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318508.495712:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580224 00000100:00100000:2.0:1767318508.495720:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580224 00000100:00100000:2.0:1767318508.495769:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 724 00000100:00100000:2.0:1767318508.495777:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982580224:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318508.495800:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318508.495849:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982580224:12345-0@lo:38:kworker.0 Request processed in 72us (317us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318508.495856:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 724 00000100:00080000:2.0:1767318508.495894:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318508.495899:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318508.495902:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982580224:0@lo:38:kworker.0 00010000:00080000:1.0:1767318508.498450:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1180 x1853163982579328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318524 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318508.498466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982579328:0@lo:400:kworker.0 00010000:00010000:1.0:1767318508.612613:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318509.639102:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318510.478759:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915572608 00000100:00100000:0.0:1767318510.478771:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915572608 00000100:00100000:0.0:1767318510.478842:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915572480 00000100:00100000:0.0:1767318510.478847:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915572480 00000100:00100000:0.0:1767318510.478870:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 559 00000100:00100000:0.0:1767318510.478880:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915572608:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318510.478962:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915572608:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (484us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318510.478970:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 559 00000100:00100000:0.0:1767318510.478988:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 725 00000100:00100000:0.0:1767318510.478992:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915572480:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318510.479012:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318510.479038:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915572480:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (504us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318510.479044:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 725 00000100:00080000:1.0:1767318510.479299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767318510.496327:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915572736 offset 0 length 520: 4 00010000:00010000:1.0:1767318511.684642:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318512.068555:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982580352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318512.068627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318512.068735:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580352 00000100:00100000:2.0:1767318512.068744:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580352 00000100:00100000:0.0:1767318512.068834:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982580480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318512.068865:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318512.069738:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580480 00000100:00100000:2.0:1767318512.069745:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580480 00000100:00100000:0.0:1767318512.070041:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982580352:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318512.073021:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318512.073306:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318512.074638:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 397 00000100:00100000:2.0:1767318512.074653:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982580480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318512.074688:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318512.074795:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982580480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 146us (5928us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318512.074803:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 397 00000100:00100000:0.0:1767318512.075178:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982580480:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318512.708536:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318513.604709:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318488/real 1767318488] req@ffff9346059cc000 x1853163982574080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318513 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318513.604748:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346059cc000 x1853163982574080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318513 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318513.604770:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318513.604779:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318513.604782:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982574080:0@lo:250:kworker.0 00000100:00100000:0.0:1767318513.605112:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318513.605132:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982580736:0@lo:400:kworker.0 00000100:00100000:0.0:1767318513.605203:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318513.605267:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318513.605307:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982581504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318513.605318:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580736 00000100:00100000:0.0:1767318513.605326:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580736 00000100:00100000:1.0:1767318513.605343:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318513.605380:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318513.605395:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982581504 00000100:00100000:1.0:1767318513.605398:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982581504 00000100:00100000:2.0:1767318513.605418:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982580608:0@lo:250:kworker.0 00000100:00100000:1.0:1767318513.605429:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 377 00000100:00100000:1.0:1767318513.605438:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982581504:12345-0@lo:400:kworker.0 00000400:00000100:2.0:1767318513.605441:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982580608 offset 0 length 520: 4 00000100:00100000:2.0:1767318513.605459:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982580864:0@lo:38:kworker.0 00000100:00100000:1.0:1767318513.605485:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982581504:12345-0@lo:400:kworker.0 Request processed in 48us (143us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318513.605493:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 377 00000100:00100000:1.0:1767318513.605515:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318513.605556:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982581504:0@lo:400:kworker.0 00000100:00100000:1.0:1767318513.605572:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982581376 00000100:00100000:1.0:1767318513.605575:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982581376 00000100:00100000:1.0:1767318513.605596:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 378 00000100:00100000:1.0:1767318513.605602:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982581376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318513.605634:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982581376:12345-0@lo:400:kworker.0 Request processed in 33us (118us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318513.605640:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 378 00000100:00100000:1.0:1767318513.605662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982581376:0@lo:400:kworker.0 00000100:00100000:0.0:1767318513.611501:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580992 00000100:00100000:0.0:1767318513.611518:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580992 00000100:00100000:0.0:1767318513.611618:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982580864 00000100:00100000:0.0:1767318513.611622:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982580864 00000100:00100000:0.0:1767318513.611641:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 560 00000100:00100000:0.0:1767318513.611655:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982580736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318513.611739:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982580736:12345-0@lo:400:kworker.0 Request processed in 86us (6535us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318513.611748:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 560 00000100:00100000:0.0:1767318513.611766:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982581120 00000100:00100000:0.0:1767318513.611770:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982581120 00000100:00100000:0.0:1767318513.611786:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 561 00000100:00100000:0.0:1767318513.611798:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982580992:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318513.611847:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982580992:12345-0@lo:400:kworker.0 Request processed in 50us (6475us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318513.611854:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 561 00000100:00100000:0.0:1767318513.611864:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982581632 00000100:00100000:0.0:1767318513.611868:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982581632 00000100:00100000:0.0:1767318513.611879:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00100000:0.0:1767318513.611884:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982580864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318513.611907:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318513.615117:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982580864:12345-0@lo:38:kworker.0 Request processed in 64us (3871us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318513.615140:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00010000:00010000:1.0:1767318514.757799:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318515.597980:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915572864 00000100:00100000:2.0:1767318515.597990:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915572864 00000100:00100000:2.0:1767318515.598073:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 729 00000100:00100000:2.0:1767318515.598081:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915572864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318515.598103:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318515.598177:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915572864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (338us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318515.598185:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 729 00000100:00100000:2.0:1767318515.608550:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318515.609156:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915572992 00000100:00100000:1.0:1767318515.609168:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915572992 00000100:00100000:1.0:1767318515.609339:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 562 00000100:00100000:1.0:1767318515.609350:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915572992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318515.610118:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915572992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (906us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318515.610132:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 562 00010000:00010000:1.0:1767318515.781160:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318517.188677:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318517.188704:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982581760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318517.188778:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318517.189150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318517.189189:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982581760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318517.189411:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318517.189420:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982581888:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318517.189447:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318517.189538:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982581888 00000100:00100000:2.0:1767318517.189543:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982581888 00000100:00100000:2.0:1767318517.189585:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 399 00000100:00100000:2.0:1767318517.189592:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982581888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318517.189609:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318517.189646:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982581888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318517.189652:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 399 00000100:00100000:2.0:1767318517.189683:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982581888:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318517.189739:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318517.828755:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318518.724563:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318518.724574:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318518.724643:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318518.724647:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318518.724651:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318518.724659:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3762 00000100:00080000:1.0:1767318518.724664:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318518.724669:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318518.724690:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53800 x1853163982582144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318518.724734:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318518.724738:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318518.724740:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318518.724743:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3762 00000100:00080000:1.0:1767318518.724746:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318518.724750:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318518.724761:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913800 x1853163982582400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318518.724769:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318518.724771:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318518.724773:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318518.724777:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3762 00000100:00080000:1.0:1767318518.724779:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318518.724782:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318518.724792:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163982582528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318518.724814:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318518.724817:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318518.724819:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318518.724822:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3762 00000100:00080000:1.0:1767318518.724825:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318518.724828:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318518.724852:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163982582912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318518.724877:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982582656:0@lo:400:kworker.0 00000100:00100000:0.0:1767318518.724895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982582144:0@lo:38:kworker.0 00000100:00100000:0.0:1767318518.724958:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318518.724959:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318518.725029:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982582016:0@lo:400:kworker.0 00000100:00100000:0.0:1767318518.727577:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982582400:0@lo:38:kworker.0 00000100:00100000:0.0:1767318518.727877:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318518.727912:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982582528:0@lo:38:kworker.0 00000100:00100000:0.0:1767318518.727946:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318518.727980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982582912:0@lo:38:kworker.0 00000100:00100000:0.0:1767318518.728005:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318518.728059:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982582272:0@lo:400:kworker.0 00000100:00100000:0.0:1767318518.728083:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318518.728122:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582144 00000100:00100000:0.0:1767318518.728130:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582144 00000100:00100000:0.0:1767318518.728165:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582400 00000100:00100000:0.0:1767318518.728169:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582400 00000100:00100000:0.0:1767318518.728180:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 730 00000100:00100000:0.0:1767318518.728187:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582144:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318518.728280:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582912 00000100:00100000:1.0:1767318518.728285:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582912 00000100:00100000:1.0:1767318518.728301:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582272 00000100:00100000:1.0:1767318518.728304:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582272 00010000:00020000:0.0:1767318518.728310:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318518.728348:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 563 00000100:00100000:1.0:1767318518.728357:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982582272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318518.728364:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582144:12345-0@lo:38:kworker.0 Request processed in 174us (3412us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318518.728374:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 730 00000100:00100000:0.0:1767318518.728395:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00100000:0.0:1767318518.728400:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318518.728410:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318518.728414:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982582272:12345-0@lo:400:kworker.0 Request processed in 57us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318518.728423:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 563 00000100:00100000:0.0:1767318518.728444:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582400:12345-0@lo:38:kworker.0 Request processed in 41us (805us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318518.728449:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00000100:00100000:0.0:1767318518.728461:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000100:00100000:0.0:1767318518.728465:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318518.728473:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318518.728493:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982582912:12345-0@lo:38:kworker.0 Request processed in 27us (488us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318518.728498:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00080000:0.0:1767318518.728521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318518.728548:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a913b80 x1853163982582272/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318534 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318518.728562:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982582272:0@lo:400:kworker.0 00000100:00080000:0.0:1767318518.728590:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318518.728595:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318518.728598:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982582144:0@lo:38:kworker.0 00000100:00080000:0.0:1767318518.728607:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318518.728611:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318518.728615:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982582400:0@lo:38:kworker.0 00000100:00080000:0.0:1767318518.728622:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318518.728625:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318518.728628:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982582912:0@lo:38:kworker.0 00000100:00100000:1.0:1767318518.728644:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582528 00000100:00100000:1.0:1767318518.728651:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582528 00000100:00100000:1.0:1767318518.728675:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 208 00000100:00100000:1.0:1767318518.728680:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982582528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318518.728691:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318518.728719:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982582528:12345-0@lo:38:kworker.0 Request processed in 38us (773us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318518.728724:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 208 00000100:00100000:1.0:1767318518.728743:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582656 00000100:00100000:1.0:1767318518.728747:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582656 00000100:00100000:1.0:1767318518.728771:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 379 00000100:00100000:1.0:1767318518.728776:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982582656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318518.728811:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982582656:12345-0@lo:400:kworker.0 Request processed in 34us (3860us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318518.728816:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 379 00000100:00080000:0.0:1767318518.728926:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318518.728931:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318518.728934:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982582528:0@lo:38:kworker.0 00000100:00100000:1.0:1767318518.729058:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982582656:0@lo:400:kworker.0 00000100:00100000:1.0:1767318518.729254:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582784 00000100:00100000:1.0:1767318518.729259:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582784 00000100:00100000:1.0:1767318518.729350:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 380 00000100:00100000:1.0:1767318518.729357:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982582784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318518.729395:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982582784:12345-0@lo:400:kworker.0 Request processed in 39us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318518.729402:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 380 00000100:00100000:0.0:1767318518.729527:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982582784:0@lo:400:kworker.0 00000100:00100000:2.0:1767318518.731009:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318518.732321:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982582016 00000100:00100000:0.0:1767318518.732334:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982582016 00000100:00100000:0.0:1767318518.732420:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 564 00000100:00100000:0.0:1767318518.732431:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982582016:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767318518.732484:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318518.732516:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982582016:12345-0@lo:400:kworker.0 Request processed in 86us (1520us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318518.732525:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 564 00010000:00080000:2.0:1767318518.732536:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f52a00 x1853163982582016/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318534 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318518.732553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982582016:0@lo:400:kworker.0 00010000:00010000:1.0:1767318518.852508:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318520.716619:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915573248 00000100:00100000:1.0:1767318520.716630:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915573248 00000100:00100000:1.0:1767318520.716713:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 565 00000100:00100000:1.0:1767318520.716724:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915573248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318520.716815:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915573248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (374us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318520.716822:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 565 00000100:00080000:2.0:1767318520.716889:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318520.724992:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318520.725092:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915573120 00000100:00100000:2.0:1767318520.725112:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915573120 00000100:00100000:2.0:1767318520.725168:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 733 00000100:00100000:2.0:1767318520.725176:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915573120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318520.725199:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318520.725308:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915573120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 130us (326us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318520.725316:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 733 00010000:00010000:1.0:1767318520.900573:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318521.925005:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318522.308658:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318522.308679:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982583168:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318522.308750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318522.308809:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583168 00000100:00100000:1.0:1767318522.308818:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583168 00000100:00100000:1.0:1767318522.308907:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 400 00000100:00100000:1.0:1767318522.308918:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982583168:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318522.308927:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767318522.308954:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318522.309012:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982583168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318522.309020:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 400 00000100:00100000:2.0:1767318522.309885:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982583168:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318522.311867:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583040 00000100:00100000:1.0:1767318522.311877:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583040 00000100:00100000:1.0:1767318522.311964:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 401 00000100:00100000:1.0:1767318522.311974:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982583040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318522.312005:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767318522.312192:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318522.312234:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982583040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 262us (1309us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318522.312242:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 401 00000100:00100000:0.0:1767318522.314458:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982583040:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318523.844413:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318523.844426:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318523.844495:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318523.844500:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318523.844504:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318523.844512:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3767 00000100:00080000:1.0:1767318523.844516:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318523.844522:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318523.844543:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7480 x1853163982583424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318523.844586:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318523.844589:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318523.844591:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318523.844595:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3767 00000100:00080000:1.0:1767318523.844597:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318523.844601:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318523.844612:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163982583680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318523.844669:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318523.844672:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318523.844674:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318523.844678:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3767 00000100:00080000:1.0:1767318523.844680:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318523.844683:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318523.844696:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5f80 x1853163982583808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318523.844715:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318523.844719:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318523.844720:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318523.844724:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3767 00000100:00080000:1.0:1767318523.844726:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318523.844729:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318523.844741:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6a00 x1853163982584192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318523.844839:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982584064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318523.844948:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318523.844999:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318523.845005:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982583296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318523.845030:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318523.845068:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982583552:0@lo:400:kworker.0 00000100:00100000:0.0:1767318523.845090:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318523.845118:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318523.845123:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982583936:0@lo:400:kworker.0 00000100:00100000:0.0:1767318523.845143:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318523.845312:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584064 00000100:00100000:1.0:1767318523.845319:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584064 00000100:00100000:1.0:1767318523.845357:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583936 00000100:00100000:1.0:1767318523.845361:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583936 00000100:00100000:1.0:1767318523.845382:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 381 00000100:00100000:1.0:1767318523.845390:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982584064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318523.845449:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982584064:12345-0@lo:400:kworker.0 Request processed in 59us (501us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318523.845456:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 381 00000100:00100000:1.0:1767318523.845475:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 382 00000100:00100000:1.0:1767318523.845482:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982583936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318523.845508:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982583936:12345-0@lo:400:kworker.0 Request processed in 27us (364us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318523.845512:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 382 00000100:00100000:1.0:1767318523.845534:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583296 00000100:00100000:1.0:1767318523.845537:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583296 00000100:00100000:1.0:1767318523.845565:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583552 00000100:00100000:1.0:1767318523.845568:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583552 00000100:00100000:1.0:1767318523.845583:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 566 00000100:00100000:1.0:1767318523.845588:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982583296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318523.845616:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982583296:12345-0@lo:400:kworker.0 Request processed in 27us (585us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318523.845622:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 566 00000100:00100000:1.0:1767318523.845633:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 567 00000100:00100000:1.0:1767318523.845639:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982583552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318523.845662:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982583552:12345-0@lo:400:kworker.0 Request processed in 23us (571us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318523.845666:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 567 00000100:00100000:0.0:1767318523.846237:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583424 00000100:00100000:0.0:1767318523.846242:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583424 00000100:00100000:0.0:1767318523.846258:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583680 00000100:00100000:0.0:1767318523.846260:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583680 00000100:00100000:0.0:1767318523.846266:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584192 00000100:00100000:0.0:1767318523.846302:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584192 00000100:00100000:0.0:1767318523.846317:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00100000:0.0:1767318523.846322:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982583424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318523.846339:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318523.846374:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982583424:12345-0@lo:38:kworker.0 Request processed in 51us (487us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318523.846381:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00000100:00100000:0.0:1767318523.846394:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00100000:0.0:1767318523.846398:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982583680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318523.846406:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318523.846427:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982583680:12345-0@lo:38:kworker.0 Request processed in 28us (497us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318523.846432:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00100000:0.0:1767318523.846442:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 736 00000100:00100000:0.0:1767318523.846446:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982584192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318523.846454:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318523.846474:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982584192:12345-0@lo:38:kworker.0 Request processed in 27us (471us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318523.846478:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 736 00000100:00100000:0.0:1767318523.846502:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982583808 00000100:00100000:0.0:1767318523.846505:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982583808 00000100:00100000:0.0:1767318523.846522:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 209 00000100:00100000:0.0:1767318523.846526:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982583808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318523.846535:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318523.846560:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982583808:12345-0@lo:38:kworker.0 Request processed in 32us (600us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318523.846565:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 209 00000100:00100000:0.0:1767318523.846599:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982583936:0@lo:400:kworker.0 00010000:00080000:0.0:1767318523.846613:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982583552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318539 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318523.846623:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982583552:0@lo:400:kworker.0 00010000:00080000:0.0:1767318523.846653:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5c00 x1853163982583296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318539 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318523.846663:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982583296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318523.846674:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982584064:0@lo:400:kworker.0 00000100:00080000:2.0:1767318523.847383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318523.973308:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318524.996518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318525.840458:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915573376 00000100:00100000:1.0:1767318525.840471:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915573376 00000100:00100000:2.0:1767318525.840498:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915573504 00000100:00100000:2.0:1767318525.840502:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915573504 00000100:00100000:1.0:1767318525.840533:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 737 00000100:00100000:1.0:1767318525.840542:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915573376:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767318525.840562:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 568 00000100:00100000:2.0:1767318525.840569:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915573504:12345-192.168.204.7@tcp:400:kworker.0 00010000:00020000:1.0:1767318525.840572:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318525.840642:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915573376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 100us (383us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318525.840650:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 737 00000100:00100000:2.0:1767318525.842756:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915573504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 2176us (2363us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318525.842771:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 568 00010000:00010000:1.0:1767318527.044506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318527.428884:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318527.428914:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982584320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318527.428991:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318527.429046:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584320 00000100:00100000:1.0:1767318527.429056:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584320 00000100:00100000:1.0:1767318527.429118:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 402 00000100:00100000:1.0:1767318527.429127:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982584320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318527.429155:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318527.429205:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982584448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318527.429205:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982584320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318527.429213:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 402 00000100:00100000:0.0:1767318527.429243:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318527.429313:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982584320:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318527.429326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318527.435675:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584448 00000100:00100000:1.0:1767318527.435690:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584448 00000100:00100000:1.0:1767318527.435763:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 403 00000100:00080000:2.0:1767318527.435803:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318527.435847:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982584448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318527.435887:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318527.435968:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982584448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 124us (6724us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318527.436047:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982584448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318527.442675:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 403 00010000:00010000:1.0:1767318528.068846:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318528.964583:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318528.964595:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318528.964682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982584576:0@lo:400:kworker.0 00000100:00100000:1.0:1767318528.964754:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318528.964877:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318528.964882:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318528.964886:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318528.964892:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3772 00000100:00080000:2.0:1767318528.964895:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318528.964900:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318528.964924:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3480 x1853163982584704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318528.964956:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318528.964960:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318528.964962:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318528.964965:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3772 00000100:00080000:2.0:1767318528.964967:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318528.964971:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318528.964980:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163982584960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318528.965037:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318528.965066:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982584704:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.965095:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.965140:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584704 00000100:00100000:0.0:1767318528.965154:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584704 00000100:00100000:0.0:1767318528.965176:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00100000:0.0:1767318528.965181:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982584704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318528.965199:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318528.965233:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982584704:12345-0@lo:38:kworker.0 Request processed in 50us (137us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318528.965239:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00000100:00080000:0.0:1767318528.965312:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318528.965317:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318528.965320:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982584704:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.965353:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982584832:0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.965378:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.965403:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584832 00000100:00100000:0.0:1767318528.965407:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584832 00000100:00100000:0.0:1767318528.965433:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 570 00000100:00100000:0.0:1767318528.965438:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982584832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.965470:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982584832:12345-0@lo:400:kworker.0 Request processed in 31us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318528.965476:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 570 00010000:00080000:0.0:1767318528.965518:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0700 x1853163982584832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318544 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318528.965528:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982584832:0@lo:400:kworker.0 00000100:00080000:0.0:1767318528.965541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318528.966469:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318528.966474:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318528.966487:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318528.966493:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3772 00000100:00080000:2.0:1767318528.966497:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318528.966503:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318528.966530:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982585088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318528.966612:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318528.966625:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982585216:0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.966669:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.966713:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585216 00000100:00100000:0.0:1767318528.966719:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585216 00000100:00100000:0.0:1767318528.966756:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 383 00000100:00100000:0.0:1767318528.966776:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982585216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.966820:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982585216:12345-0@lo:400:kworker.0 Request processed in 56us (149us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318528.966827:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 383 00000100:00100000:0.0:1767318528.966968:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982584960:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.966994:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.967019:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982585088:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.967042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.967076:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982584960 00000100:00100000:0.0:1767318528.967080:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982584960 00000100:00100000:0.0:1767318528.967099:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 739 00000100:00100000:0.0:1767318528.967104:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982584960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318528.967122:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318528.967153:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982584960:12345-0@lo:38:kworker.0 Request processed in 48us (158us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318528.967160:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 739 00000100:00100000:0.0:1767318528.967196:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982585216:0@lo:400:kworker.0 00000100:00080000:0.0:1767318528.967220:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318528.967224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318528.967227:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982584960:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.967242:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982585344:0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.969269:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318528.969374:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585344 00000100:00100000:0.0:1767318528.969383:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585344 00000100:00100000:0.0:1767318528.969433:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 384 00000100:00100000:0.0:1767318528.969443:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982585344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.969733:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982585344:12345-0@lo:400:kworker.0 Request processed in 291us (472us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318528.969742:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 384 00000100:00100000:0.0:1767318528.969799:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982585344:0@lo:400:kworker.0 00000100:00100000:0.0:1767318528.969845:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585088 00000100:00100000:0.0:1767318528.969850:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585088 00000100:00100000:0.0:1767318528.969870:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 210 00000100:00100000:0.0:1767318528.969875:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982585088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318528.969896:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318528.969932:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982585088:12345-0@lo:38:kworker.0 Request processed in 56us (2889us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318528.969938:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 210 00000100:00080000:0.0:1767318528.969969:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318528.969974:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318528.969978:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982585088:0@lo:38:kworker.0 00010000:00080000:1.0:1767318528.979984:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3100 x1853163982584576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318544 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318528.980005:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982584576:0@lo:400:kworker.0 00000100:00080000:2.0:1767318528.980130:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318528.980238:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318528.980243:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318528.980248:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3772 00000100:00080000:2.0:1767318528.980253:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318528.980259:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318528.980381:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982585472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318528.982540:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982585472:0@lo:38:kworker.0 00000100:00100000:0.0:1767318528.982630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318528.983839:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318528.983846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318528.983850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982585472:0@lo:38:kworker.0 00010000:00010000:1.0:1767318530.117006:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318530.958389:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318530.958652:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915573632 00000100:00100000:1.0:1767318530.958663:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915573632 00000100:00100000:1.0:1767318530.958717:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 741 00000100:00100000:1.0:1767318530.958734:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915573632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318530.958759:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318530.958831:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915573632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 105us (453us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318530.958838:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 741 00000100:00100000:1.0:1767318530.972906:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767318530.974289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318531.140692:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318532.552330:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318532.552340:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982585728:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318532.552369:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318532.552406:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585600 00000100:00100000:1.0:1767318532.552412:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585600 00000100:00100000:1.0:1767318532.552458:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585728 00000100:00100000:1.0:1767318532.552462:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585728 00000100:00100000:1.0:1767318532.552479:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 404 00000100:00100000:1.0:1767318532.552487:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982585600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318532.552515:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318532.552554:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982585600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (403us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318532.552565:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 404 00000100:00100000:1.0:1767318532.552583:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 405 00000100:00100000:1.0:1767318532.552588:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982585728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318532.552603:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318532.552632:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982585728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318532.552638:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 405 00000100:00080000:1.0:1767318532.552654:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318532.552658:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318532.552686:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982585728:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318533.188552:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318534.084886:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0000 x1853163982586752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318534.084909:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982586112:0@lo:400:kworker.0 00000100:00100000:3.0:1767318534.084991:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318534.085052:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318534.085058:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982586496:0@lo:400:kworker.0 00000100:00100000:3.0:1767318534.085096:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318534.085174:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982585984:0@lo:38:kworker.0 00000100:00100000:2.0:1767318534.085205:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318534.085232:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982586240:0@lo:38:kworker.0 00000100:00100000:2.0:1767318534.085256:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318534.085290:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982586368:0@lo:38:kworker.0 00000100:00100000:2.0:1767318534.085313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318534.085334:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982586752:0@lo:38:kworker.0 00000100:00100000:2.0:1767318534.085354:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318534.085399:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586112 00000100:00100000:2.0:1767318534.085406:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586112 00000100:00100000:2.0:1767318534.085447:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585984 00000100:00100000:2.0:1767318534.085449:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585984 00000100:00100000:2.0:1767318534.085457:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586240 00000100:00100000:2.0:1767318534.085459:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586240 00000100:00100000:2.0:1767318534.085464:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586752 00000100:00100000:2.0:1767318534.085466:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586752 00000100:00100000:2.0:1767318534.085476:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 572 00000100:00100000:2.0:1767318534.085484:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982586112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318534.085530:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982586112:12345-0@lo:400:kworker.0 Request processed in 47us (538us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318534.085538:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 572 00000100:00100000:2.0:1767318534.085556:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 742 00000100:00100000:2.0:1767318534.085560:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982585984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318534.085577:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318534.085609:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982585984:12345-0@lo:38:kworker.0 Request processed in 47us (403us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318534.085615:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 742 00000100:00100000:2.0:1767318534.085626:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00100000:2.0:1767318534.085630:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982586240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318534.085639:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318534.085660:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982586240:12345-0@lo:38:kworker.0 Request processed in 29us (404us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318534.085679:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00100000:2.0:1767318534.085688:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00100000:2.0:1767318534.085691:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982586752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318534.085700:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318534.085719:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982586752:12345-0@lo:38:kworker.0 Request processed in 26us (365us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318534.085724:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00010000:00080000:2.0:1767318534.085754:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0700 x1853163982586112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318550 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318534.085766:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982586112:0@lo:400:kworker.0 00000100:00080000:2.0:1767318534.085792:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318534.085796:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318534.085799:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982585984:0@lo:38:kworker.0 00000100:00080000:2.0:1767318534.085806:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318534.085809:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318534.085811:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982586240:0@lo:38:kworker.0 00000100:00080000:2.0:1767318534.085817:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318534.085820:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318534.085822:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982586752:0@lo:38:kworker.0 00000100:00100000:0.0:1767318534.086191:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982586624:0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086227:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318534.086261:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318534.086267:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982585856:0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086308:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318534.086338:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586496 00000100:00100000:0.0:1767318534.086343:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586496 00000100:00100000:0.0:1767318534.086387:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586624 00000100:00100000:0.0:1767318534.086391:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586624 00000100:00100000:0.0:1767318534.086409:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 385 00000100:00100000:0.0:1767318534.086416:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982586496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086451:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982586496:12345-0@lo:400:kworker.0 Request processed in 36us (1354us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318534.086457:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 385 00000100:00100000:0.0:1767318534.086467:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 386 00000100:00100000:0.0:1767318534.086472:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982586624:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086494:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982586624:12345-0@lo:400:kworker.0 Request processed in 23us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318534.086499:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 386 00000100:00100000:0.0:1767318534.086513:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982585856 00000100:00100000:0.0:1767318534.086516:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982585856 00000100:00100000:0.0:1767318534.086532:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 573 00000100:00100000:0.0:1767318534.086538:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982585856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086563:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982585856:12345-0@lo:400:kworker.0 Request processed in 25us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318534.086567:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 573 00000100:00100000:0.0:1767318534.086585:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586368 00000100:00100000:0.0:1767318534.086589:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586368 00000100:00100000:0.0:1767318534.086607:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 211 00000100:00100000:0.0:1767318534.086611:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982586368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318534.086622:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318534.086653:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982586368:12345-0@lo:38:kworker.0 Request processed in 40us (1339us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318534.086659:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 211 00010000:00080000:0.0:1767318534.086707:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3480 x1853163982585856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318550 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318534.086718:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982585856:0@lo:400:kworker.0 00000100:00100000:0.0:1767318534.086728:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982586624:0@lo:400:kworker.0 00000100:00080000:0.0:1767318534.086753:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318534.086756:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318534.086759:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982586368:0@lo:38:kworker.0 00000100:00100000:2.0:1767318534.087047:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982586496:0@lo:400:kworker.0 00000100:00080000:1.0:1767318534.087220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318534.087225:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318534.087227:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318534.217868:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318535.060924:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767318535.061028:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915573888 offset 0 length 520: 4 00000100:00100000:1.0:1767318535.069680:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574144 00000100:00100000:1.0:1767318535.069697:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574144 00000100:00100000:1.0:1767318535.069802:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 574 00000100:00100000:1.0:1767318535.069814:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915574144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318535.069859:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318535.069934:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574016 00000100:00100000:0.0:1767318535.069939:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574016 00000100:00080000:2.0:1767318535.069995:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318535.070335:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 745 00000100:00100000:0.0:1767318535.070344:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915574016:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318535.070375:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318535.070443:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915574144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 630us (9531us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318535.070450:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 574 00000100:00100000:0.0:1767318535.073964:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915574016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3605us (4094us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318535.073981:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 745 00010000:00010000:3.0:1767318535.236498:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318536.260679:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318537.284503:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318537.668642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982587008:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318537.668657:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318537.668671:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982586880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318537.668713:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318537.668742:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318537.668822:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587008 00000100:00100000:1.0:1767318537.668830:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587008 00000100:00100000:2.0:1767318537.668831:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982586880 00000100:00100000:2.0:1767318537.668837:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982586880 00000100:00100000:1.0:1767318537.668956:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 406 00000100:00100000:1.0:1767318537.668967:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982587008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318537.668996:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318537.669051:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982587008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318537.669058:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 406 00000100:00100000:1.0:1767318537.669106:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982587008:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318537.674774:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 407 00000100:00100000:2.0:1767318537.675310:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982586880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318537.675348:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318537.675423:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982586880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (6681us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318537.675431:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 407 00000100:00080000:3.0:1767318537.675551:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318537.675552:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982586880:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318537.675559:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318538.180966:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318513/real 1767318513] req@ffff93460a911500 x1853163982580608/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318538 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318538.181000:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a911500 x1853163982580608/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318538 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318538.181022:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318538.181049:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318538.181053:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982580608:0@lo:250:kworker.0 00000100:00080000:2.0:1767318538.181395:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318538.181401:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318538.181404:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318538.181409:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3762 00000100:00080000:2.0:1767318538.181413:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318538.181419:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318538.181450:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163982587136/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318538.181514:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982587136:0@lo:250:kworker.0 00000400:00000100:1.0:1767318538.181564:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982587136 offset 0 length 520: 4 00000100:00080000:2.0:1767318538.181671:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318538.181675:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318538.181677:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318538.181681:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3782 00000100:00080000:2.0:1767318538.181684:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318538.181688:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318538.181702:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0a80 x1853163982587392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318538.181731:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318538.181745:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982587264:0@lo:400:kworker.0 00000100:00100000:3.0:1767318538.181795:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318538.181885:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587264 00000100:00100000:0.0:1767318538.181893:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587264 00000100:00100000:0.0:1767318538.181972:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 575 00000100:00100000:0.0:1767318538.181982:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982587264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318538.182046:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982587264:12345-0@lo:400:kworker.0 Request processed in 66us (250us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318538.182054:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 575 00010000:00080000:0.0:1767318538.182102:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1180 x1853163982587264/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318554 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318538.182117:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982587264:0@lo:400:kworker.0 00000100:00080000:3.0:1767318538.182387:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318538.189439:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982587392:0@lo:38:kworker.0 00000100:00100000:1.0:1767318538.189525:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318538.189651:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587392 00000100:00100000:0.0:1767318538.189661:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587392 00000100:00100000:0.0:1767318538.189716:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00100000:0.0:1767318538.189726:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982587392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318538.189750:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318538.189829:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982587392:12345-0@lo:38:kworker.0 Request processed in 103us (302us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318538.189837:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00000100:00080000:3.0:1767318538.189936:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318538.189942:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318538.189946:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982587392:0@lo:38:kworker.0 00000100:00080000:2.0:1767318538.191901:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318538.191909:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318538.191913:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318538.191921:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3782 00000100:00080000:2.0:1767318538.191926:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318538.191934:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318538.191959:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1500 x1853163982587648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318538.191967:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767318538.191982:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767318538.191987:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982587520:0@lo:400:kworker.0 00000100:00080000:2.0:1767318538.191987:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318538.191991:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318538.191998:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3782 00000100:00080000:2.0:1767318538.192001:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318538.192005:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318538.192019:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0700 x1853163982587776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318538.192030:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318538.192033:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318538.192035:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318538.192039:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3782 00000100:00080000:2.0:1767318538.192042:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318538.192045:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318538.192058:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3480 x1853163982587904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318538.192065:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318538.192124:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587520 00000100:00100000:2.0:1767318538.192131:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587520 00000100:00100000:2.0:1767318538.192205:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 576 00000100:00100000:2.0:1767318538.192213:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982587520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318538.192254:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982587648:0@lo:38:kworker.0 00000100:00100000:3.0:1767318538.192306:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318538.192324:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982587776:0@lo:38:kworker.0 00000100:00100000:3.0:1767318538.192348:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318538.192363:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982587904:0@lo:38:kworker.0 00000100:00100000:2.0:1767318538.192383:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982587520:12345-0@lo:400:kworker.0 Request processed in 58us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318538.192384:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318538.192397:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 576 00000100:00100000:2.0:1767318538.192416:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587648 00000100:00100000:2.0:1767318538.192421:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587648 00000100:00100000:2.0:1767318538.192448:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587904 00000100:00100000:2.0:1767318538.192451:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587904 00000100:00100000:2.0:1767318538.192464:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000100:00080000:1.0:1767318538.192465:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318538.192471:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982587648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318538.192491:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:1.0:1767318538.192498:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3800 x1853163982587520/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318554 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318538.192513:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982587520:0@lo:400:kworker.0 00000100:00100000:2.0:1767318538.192529:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982587648:12345-0@lo:38:kworker.0 Request processed in 57us (221us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318538.192536:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00000100:00100000:1.0:1767318538.192546:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 748 00000100:00100000:1.0:1767318538.192553:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982587904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318538.192565:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318538.192605:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982587904:12345-0@lo:38:kworker.0 Request processed in 50us (222us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318538.192612:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 748 00000100:00100000:0.0:1767318538.192793:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982587776 00000100:00100000:0.0:1767318538.192797:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982587776 00000100:00100000:0.0:1767318538.192830:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 212 00000100:00100000:0.0:1767318538.192835:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982587776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318538.192846:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318538.192879:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982587776:12345-0@lo:38:kworker.0 Request processed in 42us (529us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318538.192886:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 212 00000100:00080000:3.0:1767318538.193058:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318538.193064:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318538.193067:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982587648:0@lo:38:kworker.0 00000100:00080000:3.0:1767318538.193077:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318538.193080:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318538.193084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982587776:0@lo:38:kworker.0 00000100:00080000:3.0:1767318538.193093:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318538.193096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318538.193099:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982587904:0@lo:38:kworker.0 00010000:00010000:3.0:1767318538.308601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318539.332771:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318540.173389:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318540.173439:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318540.173543:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574400 00000100:00100000:0.0:1767318540.173561:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574400 00000100:00100000:0.0:1767318540.173630:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574272 00000100:00100000:0.0:1767318540.173634:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574272 00000100:00100000:0.0:1767318540.173654:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 577 00000100:00100000:0.0:1767318540.173664:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915574400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318540.173749:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915574400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (372us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318540.173757:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 577 00000100:00100000:0.0:1767318540.173776:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 749 00000100:00100000:0.0:1767318540.173780:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915574272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318540.173800:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318540.173829:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915574272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (389us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318540.173835:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 749 00000100:00080000:3.0:1767318540.174346:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318540.356863:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318541.384377:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318542.404448:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318542.788457:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318542.788457:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318542.788480:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982588032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318542.788480:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982588160:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318542.791882:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318542.792154:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588032 00000100:00100000:3.0:1767318542.792165:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588032 00000100:00100000:3.0:1767318542.792244:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 408 00000100:00100000:3.0:1767318542.792256:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982588032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318542.792310:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318542.792361:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982588032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (487us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318542.792368:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 408 00000100:00080000:1.0:1767318542.792525:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318542.792582:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982588032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318542.793822:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318542.794039:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588160 00000100:00100000:0.0:1767318542.794046:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588160 00000100:00100000:0.0:1767318542.794109:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 409 00000100:00100000:0.0:1767318542.794117:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982588160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318542.794145:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318542.794188:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982588160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318542.794195:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 409 00000100:00100000:2.0:1767318542.794359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982588160:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318542.794554:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318543.300771:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318543.300782:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318543.300838:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318543.300843:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318543.300846:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318543.300856:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1767318543.300861:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318543.300867:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318543.300888:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2d80 x1853163982588416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318543.300933:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318543.300945:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318543.300947:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318543.300950:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1767318543.300952:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318543.300957:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318543.300968:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2300 x1853163982588672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318543.300969:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982588416:0@lo:38:kworker.0 00000100:00080000:0.0:1767318543.300977:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318543.300980:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318543.300981:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318543.300984:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1767318543.300987:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318543.300990:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318543.300997:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163982588800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318543.301005:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318543.301008:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318543.301010:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318543.301013:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1767318543.301016:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318543.301018:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318543.301026:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163982588928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318543.301046:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318543.301062:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982588544:0@lo:400:kworker.0 00000100:00100000:1.0:1767318543.301084:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982588672:0@lo:38:kworker.0 00000100:00100000:0.0:1767318543.301089:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318543.301106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318543.301132:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588416 00000100:00100000:1.0:1767318543.301135:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982588800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318543.301140:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588416 00000100:00100000:1.0:1767318543.301159:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318543.301175:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982588928:0@lo:38:kworker.0 00000100:00100000:0.0:1767318543.301176:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588544 00000100:00100000:0.0:1767318543.301178:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588544 00000100:00100000:0.0:1767318543.301211:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588672 00000100:00100000:0.0:1767318543.301214:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588672 00000100:00100000:0.0:1767318543.301225:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 578 00000100:00100000:0.0:1767318543.301234:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982588544:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318543.301249:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318543.301300:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588928 00000100:00100000:0.0:1767318543.301303:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982588544:12345-0@lo:400:kworker.0 Request processed in 70us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318543.301303:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588928 00000100:00100000:0.0:1767318543.301310:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 578 00000100:00100000:1.0:1767318543.301320:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00100000:1.0:1767318543.301324:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982588416:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767318543.301327:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 751 00000100:00100000:0.0:1767318543.301330:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982588672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318543.301343:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318543.301376:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982588416:12345-0@lo:38:kworker.0 Request processed in 50us (330us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318543.301382:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00100000:2.0:1767318543.301383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982588288:0@lo:400:kworker.0 00000100:00100000:1.0:1767318543.301393:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00100000:1.0:1767318543.301397:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982588928:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318543.301525:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588800 00000100:00100000:3.0:1767318543.301530:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588800 00000100:00100000:3.0:1767318543.301553:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 213 00000100:00100000:3.0:1767318543.301558:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982588800:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318543.301592:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318543.301626:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982588288 00000100:00100000:2.0:1767318543.301629:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982588288 00000100:00100000:2.0:1767318543.301652:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 579 00000100:00100000:2.0:1767318543.301658:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982588288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318543.301693:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982588288:12345-0@lo:400:kworker.0 Request processed in 35us (101us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318543.301699:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 579 00010000:00080000:2.0:1767318543.301730:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1180 x1853163982588288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318543.301742:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982588288:0@lo:400:kworker.0 00000100:00080000:2.0:1767318543.301767:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318543.301807:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1880 x1853163982588544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318543.301821:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982588544:0@lo:400:kworker.0 00010000:00020000:0.0:1767318543.307782:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318543.308381:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982588672:12345-0@lo:38:kworker.0 Request processed in 7044us (7268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318543.308395:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 751 00000100:00080000:0.0:1767318543.309234:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318543.309243:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318543.309248:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982588672:0@lo:38:kworker.0 00000100:00080000:0.0:1767318543.309264:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318543.309268:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318543.309297:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982588416:0@lo:38:kworker.0 00010000:00020000:1.0:1767318543.315159:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318543.315423:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982588928:12345-0@lo:38:kworker.0 Request processed in 14020us (14169us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318543.315517:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00080000:0.0:1767318543.317032:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318543.317042:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318543.317046:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982588928:0@lo:38:kworker.0 00010000:00020000:3.0:1767318543.318702:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318543.318810:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982588800:12345-0@lo:38:kworker.0 Request processed in 17248us (17646us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318543.318826:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 213 00000100:00080000:0.0:1767318543.318984:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318543.318990:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318543.318994:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982588800:0@lo:38:kworker.0 00010000:00010000:1.0:1767318543.428664:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318544.454721:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318545.301681:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318545.301727:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318545.301807:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574656 00000100:00100000:0.0:1767318545.301818:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574656 00000100:00100000:0.0:1767318545.301888:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574528 00000100:00100000:0.0:1767318545.301890:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574528 00000100:00100000:0.0:1767318545.301913:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 580 00000100:00100000:0.0:1767318545.301922:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915574656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318545.301998:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915574656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (328us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318545.302005:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 580 00000100:00100000:0.0:1767318545.302021:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 753 00000100:00100000:0.0:1767318545.302026:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915574528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318545.302048:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318545.302067:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915574528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (341us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318545.302076:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 753 00000100:00080000:1.0:1767318545.302594:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318545.476766:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318546.500622:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318547.524666:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318547.908826:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982589056:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318547.908926:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318547.909041:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589056 00000100:00100000:0.0:1767318547.909050:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589056 00000100:00100000:0.0:1767318547.909126:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 410 00000100:00100000:0.0:1767318547.909136:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982589056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318547.909162:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318547.909209:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982589056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318547.909215:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 410 00000100:00080000:1.0:1767318547.909286:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318547.909361:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982589056:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318547.909750:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982589184:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318547.909779:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318547.910019:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589184 00000100:00100000:0.0:1767318547.910024:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589184 00000100:00080000:1.0:1767318547.910075:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318547.910076:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 411 00000100:00100000:0.0:1767318547.910088:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982589184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318547.910111:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318547.910159:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982589184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (378us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318547.910167:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 411 00000100:00100000:1.0:1767318547.910225:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982589184:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318548.420691:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318548.420705:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318548.420778:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318548.420782:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318548.420785:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318548.420794:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3791 00000100:00080000:2.0:1767318548.420799:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318548.420807:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318548.420828:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163982589440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318548.420871:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318548.420875:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318548.420877:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318548.420881:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3791 00000100:00080000:2.0:1767318548.420883:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318548.420887:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318548.420898:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982589696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318548.420908:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767318548.420909:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767318548.420912:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318548.420916:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318548.420920:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3791 00000100:00080000:2.0:1767318548.420922:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318548.420927:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318548.420929:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982589312:0@lo:400:kworker.0 00000100:00100000:2.0:1767318548.420941:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982589824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318548.420951:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318548.420954:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318548.420956:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318548.420960:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3791 00000100:00080000:2.0:1767318548.420962:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318548.420975:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318548.420989:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1f80 x1853163982589952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318548.421003:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318548.421013:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982589568:0@lo:400:kworker.0 00000100:00100000:2.0:1767318548.421047:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318548.421259:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982589440:0@lo:38:kworker.0 00000100:00100000:3.0:1767318548.421324:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318548.423506:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982589696:0@lo:38:kworker.0 00000100:00100000:3.0:1767318548.423563:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318548.431836:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589312 00000100:00100000:1.0:1767318548.431856:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589312 00000100:00100000:1.0:1767318548.431961:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589568 00000100:00100000:1.0:1767318548.431965:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589568 00000100:00100000:1.0:1767318548.431993:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589440 00000100:00100000:1.0:1767318548.431997:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589440 00000100:00100000:1.0:1767318548.432016:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 581 00000100:00100000:1.0:1767318548.432026:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982589312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318548.432126:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982589312:12345-0@lo:400:kworker.0 Request processed in 101us (11124us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318548.432133:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 581 00000100:00100000:1.0:1767318548.432147:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 582 00000100:00100000:1.0:1767318548.432152:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982589568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318548.432179:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982589568:12345-0@lo:400:kworker.0 Request processed in 28us (11133us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318548.432182:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 582 00000100:00100000:1.0:1767318548.432190:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 754 00000100:00100000:1.0:1767318548.432193:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982589440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318548.432210:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318548.432228:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982589440:12345-0@lo:38:kworker.0 Request processed in 35us (10905us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318548.432232:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 754 00000100:00080000:1.0:1767318548.432256:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318548.432298:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0a80 x1853163982589568/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318564 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318548.432312:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982589568:0@lo:400:kworker.0 00010000:00080000:1.0:1767318548.432343:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1500 x1853163982589312/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318564 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318548.432350:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982589312:0@lo:400:kworker.0 00000100:00100000:3.0:1767318548.435366:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982589824:0@lo:38:kworker.0 00000100:00100000:3.0:1767318548.435456:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318548.435488:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982589952:0@lo:38:kworker.0 00000100:00100000:3.0:1767318548.435517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318548.435576:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318548.435582:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318548.435586:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982589440:0@lo:38:kworker.0 00000100:00100000:3.0:1767318548.435634:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589696 00000100:00100000:3.0:1767318548.435642:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589696 00000100:00100000:3.0:1767318548.435685:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589952 00000100:00100000:3.0:1767318548.435688:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589952 00000100:00100000:3.0:1767318548.435706:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 755 00000100:00100000:3.0:1767318548.435714:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982589696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318548.435740:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318548.435784:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982589696:12345-0@lo:38:kworker.0 Request processed in 70us (12220us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318548.435792:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 755 00000100:00100000:3.0:1767318548.435806:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 756 00000100:00100000:3.0:1767318548.435809:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982589952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318548.435816:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318548.435833:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982589952:12345-0@lo:38:kworker.0 Request processed in 23us (316us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318548.435838:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 756 00000100:00080000:3.0:1767318548.435864:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318548.435867:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318548.435870:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982589696:0@lo:38:kworker.0 00000100:00080000:3.0:1767318548.435878:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318548.435881:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318548.435883:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982589952:0@lo:38:kworker.0 00000100:00100000:1.0:1767318548.435964:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982589824 00000100:00100000:1.0:1767318548.436051:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982589824 00000100:00100000:1.0:1767318548.436083:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 214 00000100:00100000:1.0:1767318548.436088:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982589824:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318548.436097:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318548.436129:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982589824:12345-0@lo:38:kworker.0 Request processed in 40us (673us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318548.436134:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 214 00000100:00080000:3.0:1767318548.437166:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318548.437177:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318548.437182:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982589824:0@lo:38:kworker.0 00010000:00010000:1.0:1767318548.549079:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318549.572517:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318550.413802:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318550.413854:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318550.413920:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574912 00000100:00100000:0.0:1767318550.413931:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574912 00000100:00100000:0.0:1767318550.413998:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915574784 00000100:00100000:0.0:1767318550.414002:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915574784 00000100:00100000:0.0:1767318550.414020:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 583 00000100:00100000:0.0:1767318550.414029:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915574912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318550.414132:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915574912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (340us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318550.414140:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 583 00000100:00100000:0.0:1767318550.414161:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 757 00000100:00100000:0.0:1767318550.414166:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915574784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318550.414186:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318550.414223:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915574784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (369us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318550.414230:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 757 00000100:00080000:3.0:1767318550.419585:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318550.598408:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318551.621596:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318552.644513:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318553.028463:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982590080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318553.028550:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318553.028561:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318553.028574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982590208:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318553.028617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318553.033651:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590080 00000100:00100000:3.0:1767318553.033669:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590080 00000100:00100000:3.0:1767318553.033980:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590208 00000100:00100000:3.0:1767318553.033984:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590208 00000100:00080000:2.0:1767318553.034017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318553.034023:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318553.034759:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 412 00000100:00100000:0.0:1767318553.034775:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982590080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318553.034819:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318553.034910:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982590080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 139us (6359us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318553.034917:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 412 00000100:00100000:2.0:1767318553.035032:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982590080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318553.036087:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 413 00000100:00100000:3.0:1767318553.036097:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324e1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982590208:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318553.036122:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318553.036190:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324e1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982590208:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (7574us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318553.036198:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 413 00000100:00100000:3.0:1767318553.036504:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982590208:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318553.542919:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318553.542931:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318553.542997:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318553.543001:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318553.543005:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318553.543013:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3796 00000100:00080000:2.0:1767318553.543018:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318553.543026:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318553.543050:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318553.543051:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3480 x1853163982590464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318553.543080:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982590336:0@lo:400:kworker.0 00000100:00100000:1.0:1767318553.543154:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318553.543225:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590336 00000100:00100000:3.0:1767318553.543233:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590336 00000100:00080000:1.0:1767318553.543267:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318553.543324:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 584 00000100:00100000:3.0:1767318553.543338:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982590336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318553.543421:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982590336:12345-0@lo:400:kworker.0 Request processed in 87us (266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318553.543428:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 584 00010000:00080000:1.0:1767318553.543502:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0380 x1853163982590336/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318569 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318553.543516:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982590336:0@lo:400:kworker.0 00000100:00080000:2.0:1767318553.543626:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318553.543630:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318553.543633:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318553.543638:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3796 00000100:00080000:2.0:1767318553.543641:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318553.543646:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318553.543648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982590464:0@lo:38:kworker.0 00000100:00100000:2.0:1767318553.543662:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982590720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318553.543674:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318553.543677:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318553.543680:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767318553.543685:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318553.543689:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3796 00000100:00080000:2.0:1767318553.543692:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318553.543697:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318553.543719:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982590720:0@lo:38:kworker.0 00000100:00100000:1.0:1767318553.543741:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318553.543767:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590464 00000100:00100000:1.0:1767318553.543772:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590464 00000100:00100000:1.0:1767318553.543791:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590720 00000100:00100000:1.0:1767318553.543794:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590720 00000100:00100000:1.0:1767318553.543813:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 758 00000100:00100000:3.0:1767318553.543815:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982590592:0@lo:400:kworker.0 00000100:00100000:1.0:1767318553.543822:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982590464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318553.543843:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318553.543849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318553.543890:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590592 00000100:00100000:1.0:1767318553.543893:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982590464:12345-0@lo:38:kworker.0 Request processed in 71us (209us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318553.543895:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590592 00000100:00100000:1.0:1767318553.543904:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 758 00000100:00100000:1.0:1767318553.543922:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 759 00000100:00100000:1.0:1767318553.543926:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982590720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318553.543935:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318553.543972:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982590720:12345-0@lo:38:kworker.0 Request processed in 43us (229us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318553.544002:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1500 x1853163982590848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318553.544981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982590848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318553.545032:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318553.545084:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318553.545089:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318553.545093:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982590720:0@lo:38:kworker.0 00000100:00080000:0.0:1767318553.545104:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318553.545108:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318553.545110:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982590464:0@lo:38:kworker.0 00000100:00100000:0.0:1767318553.545148:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318553.545155:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982590976:0@lo:400:kworker.0 00000100:00100000:0.0:1767318553.545178:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318553.545200:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982591104:0@lo:400:kworker.0 00000100:00100000:0.0:1767318553.545220:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318553.545262:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590976 00000100:00100000:0.0:1767318553.545268:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590976 00000100:00100000:0.0:1767318553.545324:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591104 00000100:00100000:0.0:1767318553.545327:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591104 00000100:00100000:0.0:1767318553.545346:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 387 00000100:00100000:0.0:1767318553.545352:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982590976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318553.545395:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982590976:12345-0@lo:400:kworker.0 Request processed in 42us (215us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318553.545401:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 387 00000100:00100000:0.0:1767318553.545415:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 388 00000100:00100000:0.0:1767318553.545419:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982591104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318553.545445:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982591104:12345-0@lo:400:kworker.0 Request processed in 25us (224us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318553.545450:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 388 00000100:00100000:0.0:1767318553.545482:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982590976:0@lo:400:kworker.0 00000100:00100000:0.0:1767318553.545500:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982591104:0@lo:400:kworker.0 00000100:00080000:0.0:1767318553.545523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318553.545553:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982590848 00000100:00100000:0.0:1767318553.545556:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982590848 00000100:00100000:0.0:1767318553.545574:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 215 00000100:00100000:0.0:1767318553.545579:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982590848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318553.545593:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318553.545629:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982590848:12345-0@lo:38:kworker.0 Request processed in 49us (597us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318553.545635:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 215 00000100:00080000:0.0:1767318553.545655:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318553.545659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318553.545661:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982590848:0@lo:38:kworker.0 00000100:00100000:3.0:1767318553.550255:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 585 00000100:00100000:3.0:1767318553.550300:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982590592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318553.551878:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982590592:12345-0@lo:400:kworker.0 Request processed in 1603us (8022us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318553.551892:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 585 00000100:00080000:2.0:1767318553.552053:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318553.552059:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318553.552062:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318553.552075:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3796 00000100:00080000:2.0:1767318553.552078:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318553.552084:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318553.552116:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2680 x1853163982591232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318553.552167:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982591232:0@lo:38:kworker.0 00000100:00100000:2.0:1767318553.552212:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767318553.553223:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0000 x1853163982590592/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318569 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318553.553238:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982590592:0@lo:400:kworker.0 00000100:00100000:1.0:1767318553.553330:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 759 00000100:00100000:1.0:1767318553.553357:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591232 00000100:00100000:1.0:1767318553.553362:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591232 00000100:00100000:1.0:1767318553.553404:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 760 00000100:00100000:1.0:1767318553.553411:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982591232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318553.553432:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318553.553501:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982591232:12345-0@lo:38:kworker.0 Request processed in 89us (1288us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318553.553507:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 760 00000100:00080000:2.0:1767318553.553820:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318553.553825:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318553.553828:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982591232:0@lo:38:kworker.0 00010000:00010000:3.0:1767318553.669347:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318554.694145:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318555.532071:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318555.532159:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318555.532327:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575168 00000100:00100000:3.0:1767318555.532337:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575168 00000100:00100000:3.0:1767318555.532428:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575040 00000100:00100000:3.0:1767318555.532432:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575040 00000100:00100000:3.0:1767318555.532449:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 586 00000100:00100000:3.0:1767318555.532459:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915575168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318555.532528:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915575168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (468us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318555.532536:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 586 00000100:00100000:3.0:1767318555.532554:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 761 00000100:00100000:3.0:1767318555.532559:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915575040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318555.532578:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318555.532601:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915575040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (443us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318555.532607:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 761 00000100:00080000:0.0:1767318555.533193:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318555.716967:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318556.740776:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318557.765657:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318558.148593:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982591488:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318558.148667:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318558.148671:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318558.148683:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982591360:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318558.148722:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318558.148759:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591488 00000100:00100000:0.0:1767318558.148767:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591488 00000100:00100000:0.0:1767318558.148834:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591360 00000100:00100000:0.0:1767318558.148837:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591360 00000100:00100000:0.0:1767318558.148856:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 414 00000100:00100000:0.0:1767318558.148866:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982591488:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318558.148886:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318558.148892:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318558.148917:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 415 00000100:00100000:2.0:1767318558.148927:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982591360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318558.148953:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318558.149016:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982591360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (283us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318558.149024:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 415 00000100:00100000:3.0:1767318558.149049:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982591360:0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318558.149307:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318558.149353:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982591488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 488us (683us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318558.149361:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 414 00000100:00100000:2.0:1767318558.149394:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982591488:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318558.660719:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318558.660733:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318558.660807:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318558.660811:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318558.660816:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318558.660909:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3802 00000100:00080000:3.0:1767318558.660914:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318558.660922:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318558.660948:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163982591744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318558.661001:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318558.661005:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318558.661007:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318558.661010:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3802 00000100:00080000:3.0:1767318558.661013:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318558.661017:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318558.661029:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50e00 x1853163982592000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318558.661038:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318558.661041:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318558.661043:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318558.661047:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3802 00000100:00080000:3.0:1767318558.661049:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318558.661052:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318558.661064:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982592128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318558.661086:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318558.661089:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318558.661091:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318558.661095:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3802 00000100:00080000:3.0:1767318558.661097:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318558.661101:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318558.661111:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163982592512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318558.661249:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982591744:0@lo:38:kworker.0 00000100:00100000:0.0:1767318558.661259:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982592384:0@lo:400:kworker.0 00000100:00100000:2.0:1767318558.661321:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982591616:0@lo:400:kworker.0 00000100:00100000:3.0:1767318558.661374:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982591872:0@lo:400:kworker.0 00000100:00100000:0.0:1767318558.669041:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318558.669041:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318558.669116:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982592000:0@lo:38:kworker.0 00000100:00100000:0.0:1767318558.669155:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592384 00000100:00100000:1.0:1767318558.669161:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318558.669164:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592384 00000100:00100000:1.0:1767318558.669185:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982592128:0@lo:38:kworker.0 00000100:00100000:0.0:1767318558.669227:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 389 00000100:00100000:1.0:1767318558.669227:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318558.669239:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982592384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318558.669259:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982592512:0@lo:38:kworker.0 00000100:00100000:1.0:1767318558.669305:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318558.669332:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982592384:12345-0@lo:400:kworker.0 Request processed in 95us (303us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318558.669341:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 389 00000100:00100000:1.0:1767318558.669346:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591744 00000100:00100000:1.0:1767318558.669351:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591744 00000100:00100000:0.0:1767318558.669375:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592000 00000100:00100000:1.0:1767318558.669376:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592512 00000100:00100000:0.0:1767318558.669382:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592000 00000100:00100000:1.0:1767318558.669382:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592512 00000100:00100000:0.0:1767318558.669410:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 762 00000100:00100000:0.0:1767318558.669415:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982591744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318558.669435:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318558.669473:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982591744:12345-0@lo:38:kworker.0 Request processed in 57us (445us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318558.669479:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 762 00000100:00100000:0.0:1767318558.669494:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 763 00000100:00100000:0.0:1767318558.669498:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982592000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318558.669506:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318558.669530:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982592000:12345-0@lo:38:kworker.0 Request processed in 30us (370us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318558.669535:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 763 00000100:00100000:0.0:1767318558.669545:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 764 00000100:00100000:0.0:1767318558.669549:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982592512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318558.669570:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318558.669595:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982592512:12345-0@lo:38:kworker.0 Request processed in 45us (290us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318558.669601:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 764 00000100:00100000:1.0:1767318558.669996:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592128 00000100:00100000:1.0:1767318558.670002:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592128 00000100:00100000:1.0:1767318558.670028:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 216 00000100:00100000:0.0:1767318558.670034:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982592384:0@lo:400:kworker.0 00000100:00100000:1.0:1767318558.670035:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982592128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318558.670051:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318558.670084:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982592128:12345-0@lo:38:kworker.0 Request processed in 49us (860us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318558.670092:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 216 00000100:00080000:0.0:1767318558.670111:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318558.670118:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318558.670122:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982591744:0@lo:38:kworker.0 00000100:00100000:1.0:1767318558.670132:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982592256:0@lo:400:kworker.0 00000100:00080000:0.0:1767318558.670145:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318558.670152:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318558.670156:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982592000:0@lo:38:kworker.0 00000100:00100000:1.0:1767318558.670166:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318558.670169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318558.670175:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318558.670178:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982592128:0@lo:38:kworker.0 00000100:00080000:0.0:1767318558.670191:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318558.670204:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318558.670208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982592512:0@lo:38:kworker.0 00000100:00100000:1.0:1767318558.670210:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592256 00000100:00100000:1.0:1767318558.670215:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592256 00000100:00100000:1.0:1767318558.670247:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 390 00000100:00100000:1.0:1767318558.670255:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982592256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318558.670316:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982592256:12345-0@lo:400:kworker.0 Request processed in 63us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318558.670323:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 390 00000100:00100000:1.0:1767318558.670344:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982592256:0@lo:400:kworker.0 00000100:00100000:3.0:1767318558.672748:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318558.672897:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591872 00000100:00100000:1.0:1767318558.672905:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591872 00000100:00100000:1.0:1767318558.672974:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 587 00000100:00100000:1.0:1767318558.672984:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982591872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318558.673047:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982591872:12345-0@lo:400:kworker.0 Request processed in 64us (306us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318558.673054:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 587 00010000:00080000:3.0:1767318558.673155:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f51500 x1853163982591872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318574 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318558.673169:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982591872:0@lo:400:kworker.0 00000100:00080000:0.0:1767318558.673255:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318558.673706:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318558.673751:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982591616 00000100:00100000:3.0:1767318558.673757:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982591616 00000100:00100000:3.0:1767318558.673845:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 588 00000100:00100000:3.0:1767318558.673854:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982591616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318558.673896:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982591616:12345-0@lo:400:kworker.0 Request processed in 44us (191us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318558.673903:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 588 00010000:00080000:2.0:1767318558.673957:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a2d80 x1853163982591616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318574 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318558.673969:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982591616:0@lo:400:kworker.0 00000100:00080000:0.0:1767318558.673983:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318558.788521:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318559.813152:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318560.663373:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318560.663421:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318560.663864:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575296 00000100:00100000:0.0:1767318560.663879:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575296 00000100:00100000:0.0:1767318560.663940:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575424 00000100:00100000:0.0:1767318560.663944:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575424 00000100:00100000:0.0:1767318560.664002:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 589 00000100:00100000:0.0:1767318560.664015:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915575424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318560.664119:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915575424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 106us (696us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318560.664127:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 589 00000100:00100000:0.0:1767318560.664146:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 765 00000100:00100000:0.0:1767318560.664151:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318560.664183:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318560.664210:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (850us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318560.664217:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 765 00000100:00080000:0.0:1767318560.664235:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767318560.665765:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915575552 offset 0 length 520: 4 00010000:00010000:1.0:1767318560.838441:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318561.866619:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318562.887075:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318563.268560:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318563.268586:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982592640:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318563.268672:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318563.268785:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592640 00000100:00100000:1.0:1767318563.268794:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592640 00000100:00100000:1.0:1767318563.268855:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 416 00000100:00100000:1.0:1767318563.268864:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982592640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318563.268892:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318563.268936:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982592640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318563.268942:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 416 00000100:00080000:2.0:1767318563.269001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318563.269038:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982592640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318563.273978:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982592768:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318563.274063:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318563.274328:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592768 00000100:00100000:2.0:1767318563.274337:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592768 00000100:00100000:2.0:1767318563.274401:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 417 00000100:00100000:2.0:1767318563.274411:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982592768:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318563.274655:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767318563.274751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318563.275545:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982592768:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1132us (1478us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318563.275564:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 417 00000100:00100000:3.0:1767318563.275888:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982592768:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767318563.780395:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318563.780405:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318563.780473:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318563.780477:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318563.780480:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318563.780489:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3807 00000100:00080000:3.0:1767318563.780495:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318563.780501:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318563.780523:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5500 x1853163982593024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318563.780571:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318563.780575:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318563.780577:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318563.780580:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3807 00000100:00080000:3.0:1767318563.780582:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318563.780585:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318563.780598:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396df80 x1853163982593280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318563.780607:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318563.780610:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318563.780612:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318563.780615:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3807 00000100:00080000:3.0:1767318563.780617:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318563.780621:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318563.780634:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989880 x1853163982593408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318563.780654:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318563.780658:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318563.780659:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318563.780662:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3807 00000100:00080000:3.0:1767318563.780664:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318563.780667:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318563.780678:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35f80 x1853163982593792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318563.780711:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982593536:0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.780773:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318563.780835:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318563.780841:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982593152:0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.780867:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318563.780894:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593152 00000100:00100000:3.0:1767318563.780901:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593152 00000100:00100000:3.0:1767318563.780965:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 590 00000100:00100000:3.0:1767318563.780976:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982593152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.781023:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982593152:12345-0@lo:400:kworker.0 Request processed in 50us (154us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318563.781031:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 590 00000100:00080000:3.0:1767318563.781066:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767318563.781130:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c700 x1853163982593152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318579 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318563.781143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982593152:0@lo:400:kworker.0 00000100:00100000:2.0:1767318563.781244:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318538/real 1767318538] req@ffff9346392c3100 x1853163982587136/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318563 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318563.781264:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346392c3100 x1853163982587136/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318563 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318563.781312:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982593024:0@lo:38:kworker.0 00000100:00100000:2.0:1767318563.781339:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318563.781360:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982593280:0@lo:38:kworker.0 00000100:00100000:2.0:1767318563.781381:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318563.781396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982593408:0@lo:38:kworker.0 00000100:00100000:2.0:1767318563.781419:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318563.781441:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982593792:0@lo:38:kworker.0 00000100:00100000:2.0:1767318563.781461:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318563.781475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318563.781488:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318563.781491:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982587136:0@lo:250:kworker.0 00000100:00100000:2.0:1767318563.781519:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593024 00000100:00100000:2.0:1767318563.781523:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593024 00000100:00100000:2.0:1767318563.781541:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593280 00000100:00100000:2.0:1767318563.781544:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593280 00000100:00100000:2.0:1767318563.781552:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593792 00000100:00100000:2.0:1767318563.781555:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593792 00000100:00100000:2.0:1767318563.781565:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 766 00000100:00100000:2.0:1767318563.781570:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318563.781586:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318563.781619:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593024:12345-0@lo:38:kworker.0 Request processed in 48us (279us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318563.781629:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 766 00000100:00100000:2.0:1767318563.781642:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 767 00000100:00100000:2.0:1767318563.781645:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318563.781655:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318563.781679:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593280:12345-0@lo:38:kworker.0 Request processed in 32us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318563.781684:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 767 00000100:00100000:2.0:1767318563.781695:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 768 00000100:00100000:2.0:1767318563.781698:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318563.781706:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318563.781730:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982593792:12345-0@lo:38:kworker.0 Request processed in 30us (268us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318563.781735:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 768 00000100:00080000:2.0:1767318563.781757:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318563.781761:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318563.781764:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982593024:0@lo:38:kworker.0 00000100:00080000:2.0:1767318563.781772:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318563.781775:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318563.781778:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982593280:0@lo:38:kworker.0 00000100:00080000:2.0:1767318563.781785:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318563.781789:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318563.781792:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982593792:0@lo:38:kworker.0 00000100:00080000:2.0:1767318563.781811:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318563.781816:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318563.781817:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318563.781821:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3786 00000100:00080000:2.0:1767318563.781823:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318563.781827:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318563.781839:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982593920/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318563.781865:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982593920:0@lo:250:kworker.0 00000400:00000100:2.0:1767318563.781885:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982593920 offset 0 length 520: 4 00000100:00100000:1.0:1767318563.782217:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982592896:0@lo:400:kworker.0 00000100:00100000:1.0:1767318563.782248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318563.782335:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982592896 00000100:00100000:3.0:1767318563.782340:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982592896 00000100:00100000:3.0:1767318563.782418:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 591 00000100:00100000:3.0:1767318563.782424:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982592896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.782455:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982592896:12345-0@lo:400:kworker.0 Request processed in 31us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318563.782460:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 591 00000100:00100000:3.0:1767318563.782491:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593536 00000100:00100000:3.0:1767318563.782495:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593536 00000100:00100000:3.0:1767318563.782517:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 391 00000100:00100000:3.0:1767318563.782523:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982593536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.782553:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982593536:12345-0@lo:400:kworker.0 Request processed in 30us (1780us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318563.782558:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 391 00000100:00100000:3.0:1767318563.782582:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982593536:0@lo:400:kworker.0 00000100:00100000:3.0:1767318563.782606:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593408 00000100:00100000:3.0:1767318563.782610:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593408 00000100:00100000:3.0:1767318563.782626:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 217 00000100:00100000:3.0:1767318563.782630:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982593408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318563.782640:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318563.782670:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982593408:12345-0@lo:38:kworker.0 Request processed in 38us (1250us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318563.782676:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 217 00000100:00080000:3.0:1767318563.782697:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318563.782700:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318563.782703:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982593408:0@lo:38:kworker.0 00010000:00080000:1.0:1767318563.783227:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee7b80 x1853163982592896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318579 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318563.783239:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982592896:0@lo:400:kworker.0 00000100:00100000:1.0:1767318563.783254:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318563.783260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982593664:0@lo:400:kworker.0 00000100:00100000:1.0:1767318563.783315:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318563.783339:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982593664 00000100:00100000:1.0:1767318563.783343:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982593664 00000100:00100000:1.0:1767318563.783365:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 392 00000100:00100000:1.0:1767318563.783370:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982593664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318563.783404:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982593664:12345-0@lo:400:kworker.0 Request processed in 34us (89us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318563.783410:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 392 00000100:00100000:1.0:1767318563.783435:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982593664:0@lo:400:kworker.0 00000100:00080000:2.0:1767318563.784546:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318563.908606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318564.934902:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318565.771646:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318565.771756:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575680 00000100:00100000:0.0:1767318565.771765:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575680 00000100:00100000:0.0:1767318565.771819:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 769 00000100:00100000:0.0:1767318565.771827:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318565.771849:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318565.771910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (275us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318565.771916:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 769 00000100:00100000:3.0:1767318565.772397:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318565.772450:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575808 00000100:00100000:1.0:1767318565.772456:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575808 00000100:00100000:1.0:1767318565.772505:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 592 00000100:00100000:1.0:1767318565.772512:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915575808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318565.772560:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915575808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 48us (165us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318565.772566:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 592 00000100:00080000:2.0:1767318565.772716:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318565.956486:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318566.980490:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318568.008616:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318568.389164:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982594048:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318568.389243:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318568.389304:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594048 00000100:00100000:3.0:1767318568.389386:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594048 00000100:00100000:3.0:1767318568.389499:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 418 00000100:00100000:3.0:1767318568.389510:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982594048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318568.389545:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767318568.389608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318568.390968:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318568.390984:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982594176:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318568.391048:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318568.391920:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594176 00000100:00100000:0.0:1767318568.391929:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594176 00000100:00100000:0.0:1767318568.392000:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 419 00000100:00100000:0.0:1767318568.392021:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982594176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318568.392053:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318568.392113:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982594176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (1064us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318568.392119:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 419 00000100:00100000:0.0:1767318568.392175:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982594176:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318568.392254:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318568.396341:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982594048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 6821us (7086us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318568.396370:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 418 00000100:00100000:2.0:1767318568.405320:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982594048:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767318568.900572:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318568.900584:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318568.900666:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318568.900670:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318568.900675:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318568.900684:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3812 00000100:00080000:3.0:1767318568.900690:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318568.900699:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318568.900722:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989880 x1853163982594432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318568.900754:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318568.900758:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318568.900760:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318568.900764:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3812 00000100:00080000:3.0:1767318568.900767:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318568.900770:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318568.900783:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396df80 x1853163982594688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318568.900792:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318568.900795:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318568.900797:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318568.900800:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3812 00000100:00080000:3.0:1767318568.900802:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318568.900805:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318568.900818:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c700 x1853163982594816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318568.900846:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318568.900850:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318568.900852:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318568.900855:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3812 00000100:00080000:3.0:1767318568.900857:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318568.900861:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318568.900871:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9880 x1853163982595200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318568.900907:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982594560:0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.900985:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318568.901058:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318568.901064:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982594944:0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.901093:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318568.901125:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594944 00000100:00100000:3.0:1767318568.901134:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594944 00000100:00100000:3.0:1767318568.901181:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 393 00000100:00100000:3.0:1767318568.901189:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982594944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.901235:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982594944:12345-0@lo:400:kworker.0 Request processed in 48us (142us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318568.901365:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 393 00000100:00100000:3.0:1767318568.901413:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982594944:0@lo:400:kworker.0 00000100:00100000:2.0:1767318568.901498:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982594432:0@lo:38:kworker.0 00000100:00100000:2.0:1767318568.901529:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318568.901559:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982594688:0@lo:38:kworker.0 00000100:00100000:2.0:1767318568.901580:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318568.901595:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982594816:0@lo:38:kworker.0 00000100:00100000:2.0:1767318568.901618:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318568.901635:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982595200:0@lo:38:kworker.0 00000100:00100000:2.0:1767318568.901654:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318568.901681:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594816 00000100:00100000:2.0:1767318568.901685:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594816 00000100:00100000:2.0:1767318568.901706:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 218 00000100:00100000:2.0:1767318568.901711:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982594816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318568.901729:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318568.901764:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982594816:12345-0@lo:38:kworker.0 Request processed in 51us (144us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318568.901770:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 218 00000100:00080000:2.0:1767318568.901795:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318568.901800:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318568.901803:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982594816:0@lo:38:kworker.0 00000100:00100000:1.0:1767318568.901891:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982595072:0@lo:400:kworker.0 00000100:00100000:1.0:1767318568.901919:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318568.901947:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318568.901954:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982594304:0@lo:400:kworker.0 00000100:00100000:1.0:1767318568.901977:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318568.902044:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595072 00000100:00100000:2.0:1767318568.902049:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595072 00000100:00100000:2.0:1767318568.902084:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 394 00000100:00100000:2.0:1767318568.902091:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982595072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318568.902127:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982595072:12345-0@lo:400:kworker.0 Request processed in 36us (206us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318568.902133:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 394 00000100:00100000:2.0:1767318568.902160:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982595072:0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.902442:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594560 00000100:00100000:3.0:1767318568.902449:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594560 00000100:00100000:3.0:1767318568.902490:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594432 00000100:00100000:3.0:1767318568.902493:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594432 00000100:00100000:3.0:1767318568.902503:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594688 00000100:00100000:3.0:1767318568.902505:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594688 00000100:00100000:3.0:1767318568.902514:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595200 00000100:00100000:3.0:1767318568.902516:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595200 00000100:00100000:3.0:1767318568.902524:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982594304 00000100:00100000:3.0:1767318568.902526:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982594304 00000100:00100000:3.0:1767318568.902542:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 593 00000100:00100000:3.0:1767318568.902549:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982594560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.902604:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982594560:12345-0@lo:400:kworker.0 Request processed in 55us (1619us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318568.902611:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 593 00000100:00100000:3.0:1767318568.902624:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 594 00000100:00100000:3.0:1767318568.902629:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982594304:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318568.902663:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982594304:12345-0@lo:400:kworker.0 Request processed in 34us (685us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318568.902673:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 594 00000100:00100000:3.0:1767318568.902684:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 770 00000100:00100000:3.0:1767318568.902688:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982594432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318568.902700:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318568.902724:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982594432:12345-0@lo:38:kworker.0 Request processed in 35us (1194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318568.902731:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 770 00000100:00100000:3.0:1767318568.902741:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 771 00000100:00100000:3.0:1767318568.902744:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982594688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318568.902753:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318568.902775:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982594688:12345-0@lo:38:kworker.0 Request processed in 29us (1193us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318568.902780:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 771 00000100:00100000:3.0:1767318568.902789:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 772 00000100:00100000:3.0:1767318568.902792:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982595200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318568.902799:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318568.902818:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982595200:12345-0@lo:38:kworker.0 Request processed in 24us (1163us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318568.902823:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 772 00000100:00080000:1.0:1767318568.902903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318568.902965:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198a300 x1853163982594304/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318584 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318568.902977:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982594304:0@lo:400:kworker.0 00010000:00080000:2.0:1767318568.903017:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631989500 x1853163982594560/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318584 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318568.903026:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982594560:0@lo:400:kworker.0 00000100:00080000:2.0:1767318568.903047:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318568.903052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318568.903055:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982594432:0@lo:38:kworker.0 00000100:00080000:2.0:1767318568.903062:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318568.903065:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318568.903068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982594688:0@lo:38:kworker.0 00000100:00080000:2.0:1767318568.903075:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318568.903079:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318568.903082:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982595200:0@lo:38:kworker.0 00010000:00010000:3.0:1767318569.030321:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318570.053206:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318570.894653:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318570.897448:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915575936 00000100:00100000:0.0:1767318570.897465:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915575936 00000100:00100000:0.0:1767318570.897525:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 773 00000100:00100000:0.0:1767318570.897535:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575936:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318570.897836:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318570.898330:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915575936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 793us (3684us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318570.898340:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 773 00000100:00100000:3.0:1767318570.899257:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318570.899921:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576064 00000100:00100000:0.0:1767318570.899932:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576064 00000100:00100000:0.0:1767318570.900346:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 595 00000100:00100000:0.0:1767318570.900358:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915576064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318570.900456:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915576064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (1201us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318570.900465:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 595 00000100:00080000:2.0:1767318570.903222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318571.078883:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318572.100497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318573.127946:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318573.509362:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318573.509387:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982595328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318573.509463:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318573.512433:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318573.512470:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982595456:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318573.512549:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318573.520767:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595328 00000100:00100000:1.0:1767318573.520786:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595328 00000100:00100000:1.0:1767318573.520880:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595456 00000100:00100000:1.0:1767318573.520884:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595456 00000100:00100000:1.0:1767318573.520938:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 420 00000100:00100000:1.0:1767318573.520949:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982595328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318573.520989:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318573.521098:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982595328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 151us (11635us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318573.521106:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 420 00000100:00100000:1.0:1767318573.521127:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 421 00000100:00100000:1.0:1767318573.521135:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982595456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318573.521152:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318573.521183:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982595456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (8634us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318573.521189:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 421 00000100:00100000:1.0:1767318573.521265:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982595328:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318573.521578:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982595456:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318573.521651:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318573.521656:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318574.020850:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318574.020861:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318574.020940:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318574.020945:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318574.020949:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318574.020957:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3817 00000100:00080000:3.0:1767318574.020961:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318574.020977:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318574.021003:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f5880 x1853163982595712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318574.021034:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318574.021037:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318574.021039:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318574.021043:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3817 00000100:00080000:3.0:1767318574.021046:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318574.021049:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318574.021060:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346309f5c00 x1853163982595968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318574.021069:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318574.021073:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318574.021075:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318574.021078:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3817 00000100:00080000:3.0:1767318574.021081:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318574.021084:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318574.021096:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982596096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318574.021105:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318574.021108:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318574.021110:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318574.021112:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3817 00000100:00080000:3.0:1767318574.021114:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318574.021117:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318574.021129:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d880 x1853163982596224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318574.021154:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318574.021166:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982595584:0@lo:400:kworker.0 00000100:00100000:3.0:1767318574.021233:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318574.021458:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982595712:0@lo:38:kworker.0 00000100:00100000:2.0:1767318574.021494:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318574.021523:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982595968:0@lo:38:kworker.0 00000100:00100000:2.0:1767318574.021546:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318574.021560:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982596096:0@lo:38:kworker.0 00000100:00100000:2.0:1767318574.021583:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318574.021621:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982596224:0@lo:38:kworker.0 00000100:00100000:1.0:1767318574.021625:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346309f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982595840:0@lo:400:kworker.0 00000100:00100000:2.0:1767318574.021651:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318574.021671:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318574.021830:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595584 00000100:00100000:0.0:1767318574.021838:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595584 00000100:00100000:0.0:1767318574.021893:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595712 00000100:00100000:0.0:1767318574.021905:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595712 00000100:00100000:0.0:1767318574.021917:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595968 00000100:00100000:0.0:1767318574.021919:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595968 00000100:00100000:0.0:1767318574.021927:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596224 00000100:00100000:0.0:1767318574.021929:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596224 00000100:00100000:0.0:1767318574.021936:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982595840 00000100:00100000:0.0:1767318574.021938:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982595840 00000100:00100000:0.0:1767318574.021956:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 596 00000100:00100000:0.0:1767318574.021964:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982595584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318574.022038:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982595584:12345-0@lo:400:kworker.0 Request processed in 73us (803us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318574.022047:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 596 00000100:00100000:0.0:1767318574.022069:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 597 00010000:00080000:1.0:1767318574.022075:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d91c00 x1853163982595584/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318590 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318574.022078:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982595840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318574.022098:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982595584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318574.022126:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982595840:12345-0@lo:400:kworker.0 Request processed in 50us (456us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318574.022133:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 597 00000100:00100000:0.0:1767318574.022147:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 774 00000100:00100000:0.0:1767318574.022152:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982595712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318574.022171:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318574.022205:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982595712:12345-0@lo:38:kworker.0 Request processed in 51us (709us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318574.022211:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 774 00000100:00100000:0.0:1767318574.022223:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 775 00000100:00100000:0.0:1767318574.022228:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982595968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318574.022238:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:1.0:1767318574.022252:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346309f6d80 x1853163982595840/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318590 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318574.022263:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982595840:0@lo:400:kworker.0 00000100:00100000:0.0:1767318574.022296:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982595968:12345-0@lo:38:kworker.0 Request processed in 38us (719us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318574.022304:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 775 00000100:00100000:0.0:1767318574.022320:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 776 00000100:00100000:0.0:1767318574.022325:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982596224:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767318574.022327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318574.022334:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767318574.022336:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318574.022340:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982595712:0@lo:38:kworker.0 00000100:00080000:1.0:1767318574.022353:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767318574.022366:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982596224:12345-0@lo:38:kworker.0 Request processed in 39us (714us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318574.022372:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 776 00000100:00080000:2.0:1767318574.022440:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318574.022452:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318574.022458:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346309f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982595968:0@lo:38:kworker.0 00000100:00080000:1.0:1767318574.022472:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318574.022475:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318574.022478:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982596224:0@lo:38:kworker.0 00000100:00100000:3.0:1767318574.022697:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596096 00000100:00100000:3.0:1767318574.022703:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596096 00000100:00100000:3.0:1767318574.022737:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 219 00000100:00100000:3.0:1767318574.022743:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982596096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318574.022753:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318574.022798:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982596096:12345-0@lo:38:kworker.0 Request processed in 54us (1213us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318574.022804:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 219 00000100:00080000:1.0:1767318574.023001:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318574.023005:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318574.023009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982596096:0@lo:38:kworker.0 00010000:00010000:1.0:1767318574.151496:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318575.173152:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318576.012398:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318576.012978:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576192 00000100:00100000:0.0:1767318576.012990:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576192 00000100:00100000:0.0:1767318576.013046:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 777 00000100:00100000:0.0:1767318576.013054:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915576192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318576.013076:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318576.013148:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915576192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (760us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318576.013155:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 777 00000100:00100000:2.0:1767318576.015712:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318576.019098:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576320 00000100:00100000:3.0:1767318576.019116:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576320 00000100:00100000:3.0:1767318576.019225:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 598 00000100:00100000:3.0:1767318576.019236:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915576320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318576.019371:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915576320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 137us (3667us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318576.019379:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 598 00000100:00080000:2.0:1767318576.021543:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318576.197572:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318577.220547:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318578.244998:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318578.628677:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318578.628708:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982596352:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318578.628797:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318578.629085:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596352 00000100:00100000:0.0:1767318578.629094:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596352 00000100:00100000:0.0:1767318578.629164:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 422 00000100:00100000:0.0:1767318578.629174:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982596352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318578.629206:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318578.629254:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982596352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (457us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318578.629292:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 422 00000100:00100000:3.0:1767318578.629468:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982596352:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318578.629536:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318578.629668:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982596480:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318578.629701:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318578.629779:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596480 00000100:00100000:3.0:1767318578.629785:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596480 00000100:00100000:3.0:1767318578.629843:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 423 00000100:00100000:3.0:1767318578.629851:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982596480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318578.629867:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318578.629905:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982596480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (202us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318578.629911:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 423 00000100:00100000:3.0:1767318578.629944:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982596480:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318578.630041:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318579.140394:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318579.140405:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318579.140473:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318579.140477:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318579.140481:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318579.140488:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3822 00000100:00080000:3.0:1767318579.140494:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318579.140501:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318579.140524:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56680 x1853163982596736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318579.140580:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318579.140583:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318579.140585:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318579.140588:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3822 00000100:00080000:3.0:1767318579.140591:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318579.140594:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318579.140605:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377b80 x1853163982596992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318579.140613:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318579.140616:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318579.140618:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318579.140622:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3822 00000100:00080000:3.0:1767318579.140624:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318579.140627:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318579.140641:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604376300 x1853163982597120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318579.140649:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318579.140652:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318579.140666:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318579.140669:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3822 00000100:00080000:3.0:1767318579.140672:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318579.140675:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318579.140688:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982597248/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318579.140732:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982596864:0@lo:400:kworker.0 00000100:00100000:3.0:1767318579.140810:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318579.140980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982596736:0@lo:38:kworker.0 00000100:00100000:2.0:1767318579.141013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318579.141047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982596992:0@lo:38:kworker.0 00000100:00100000:2.0:1767318579.141068:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318579.141083:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982597120:0@lo:38:kworker.0 00000100:00100000:2.0:1767318579.141106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318579.141147:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982597248:0@lo:38:kworker.0 00000100:00100000:2.0:1767318579.141169:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318579.141199:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597120 00000100:00100000:2.0:1767318579.141207:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597120 00000100:00100000:2.0:1767318579.141238:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 220 00000100:00100000:2.0:1767318579.141245:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982597120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318579.141264:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318579.141336:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982597120:12345-0@lo:38:kworker.0 Request processed in 90us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318579.141344:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 220 00000100:00080000:2.0:1767318579.141386:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318579.141391:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318579.141393:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982597120:0@lo:38:kworker.0 00000100:00100000:0.0:1767318579.141634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982596608:0@lo:400:kworker.0 00000100:00100000:0.0:1767318579.141666:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318579.141708:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596864 00000100:00100000:0.0:1767318579.141713:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596864 00000100:00100000:0.0:1767318579.141747:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596736 00000100:00100000:0.0:1767318579.141750:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596736 00000100:00100000:0.0:1767318579.141758:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596992 00000100:00100000:0.0:1767318579.141761:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596992 00000100:00100000:0.0:1767318579.141767:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597248 00000100:00100000:0.0:1767318579.141769:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597248 00000100:00100000:0.0:1767318579.141776:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982596608 00000100:00100000:0.0:1767318579.141778:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982596608 00000100:00100000:0.0:1767318579.141792:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 599 00000100:00100000:0.0:1767318579.141808:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982596864:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318579.141852:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982596864:12345-0@lo:400:kworker.0 Request processed in 54us (1041us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318579.141858:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 599 00000100:00100000:0.0:1767318579.141872:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 600 00000100:00100000:0.0:1767318579.141878:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982596608:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318579.141923:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982596608:12345-0@lo:400:kworker.0 Request processed in 44us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318579.141929:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 600 00000100:00100000:0.0:1767318579.141941:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 778 00000100:00100000:0.0:1767318579.141945:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982596736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318579.141956:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318579.141985:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982596736:12345-0@lo:38:kworker.0 Request processed in 37us (970us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318579.141990:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 778 00000100:00100000:0.0:1767318579.142002:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 779 00000100:00100000:0.0:1767318579.142005:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982596992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318579.142014:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318579.142037:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982596992:12345-0@lo:38:kworker.0 Request processed in 30us (968us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318579.142043:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 779 00000100:00100000:0.0:1767318579.142053:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 780 00000100:00100000:0.0:1767318579.142057:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982597248:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318579.142066:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318579.142088:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982597248:12345-0@lo:38:kworker.0 Request processed in 29us (917us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318579.142093:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 780 00010000:00080000:0.0:1767318579.142119:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54a80 x1853163982596608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318595 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318579.142304:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318579.142308:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318579.142311:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982596736:0@lo:38:kworker.0 00000100:00080000:2.0:1767318579.142319:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318579.142322:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318579.142325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982596992:0@lo:38:kworker.0 00000100:00080000:2.0:1767318579.142333:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318579.142336:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318579.142339:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982597248:0@lo:38:kworker.0 00000100:00100000:0.0:1767318579.142417:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982596608:0@lo:400:kworker.0 00010000:00080000:3.0:1767318579.142515:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57480 x1853163982596864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318595 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318579.142525:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982596864:0@lo:400:kworker.0 00000100:00080000:1.0:1767318579.143091:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318579.272989:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318580.295382:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318581.133041:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318581.133094:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318581.133312:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576576 00000100:00100000:2.0:1767318581.133323:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576576 00000100:00100000:2.0:1767318581.133403:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576448 00000100:00100000:2.0:1767318581.133407:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576448 00000100:00100000:2.0:1767318581.133431:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 601 00000100:00100000:2.0:1767318581.133441:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915576576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318581.133583:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915576576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 143us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318581.133591:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 601 00000100:00100000:2.0:1767318581.133609:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 781 00000100:00100000:2.0:1767318581.133614:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915576448:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318581.133632:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318581.133659:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915576448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (565us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318581.133666:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 781 00000100:00080000:1.0:1767318581.134577:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318581.316685:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318582.340781:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318583.364665:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318583.748893:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982597376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318583.748988:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318583.749384:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597376 00000100:00100000:2.0:1767318583.749394:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597376 00000100:00080000:1.0:1767318583.749468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318583.749478:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 424 00000100:00100000:2.0:1767318583.749488:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982597376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318583.749522:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318583.749694:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982597504:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318583.749764:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318583.749881:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982597376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 393us (890us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318583.749889:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 424 00000100:00100000:3.0:1767318583.749890:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597504 00000100:00100000:3.0:1767318583.749897:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597504 00000100:00100000:3.0:1767318583.749949:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 425 00000100:00100000:3.0:1767318583.749957:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982597504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318583.749976:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318583.750016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982597376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318583.750020:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982597504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 63us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318583.750029:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 425 00000100:00100000:3.0:1767318583.750066:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982597504:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318583.750126:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318584.260931:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318584.260941:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318584.261009:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318584.261014:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318584.261018:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318584.261028:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3827 00000100:00080000:3.0:1767318584.261033:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318584.261041:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318584.261069:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960e00 x1853163982597760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318584.261114:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318584.261117:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318584.261119:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318584.261123:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3827 00000100:00080000:3.0:1767318584.261125:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318584.261129:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318584.261138:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163982598016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318584.261146:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318584.261149:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318584.261150:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318584.261153:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3827 00000100:00080000:3.0:1767318584.261155:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318584.261158:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318584.261171:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163982598144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318584.261179:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318584.261183:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318584.261185:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318584.261188:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3827 00000100:00080000:3.0:1767318584.261190:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318584.261193:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318584.261203:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8e00 x1853163982598272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318584.261233:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982597888:0@lo:400:kworker.0 00000100:00100000:3.0:1767318584.261338:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318584.261576:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982597632:0@lo:400:kworker.0 00000100:00100000:0.0:1767318584.261601:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318584.261640:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597888 00000100:00100000:0.0:1767318584.261649:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597888 00000100:00100000:0.0:1767318584.261745:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 602 00000100:00100000:0.0:1767318584.261755:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982597888:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318584.261814:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982597888:12345-0@lo:400:kworker.0 Request processed in 61us (475us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318584.261821:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 602 00000100:00100000:1.0:1767318584.261830:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597632 00000100:00100000:1.0:1767318584.261834:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597632 00000100:00080000:0.0:1767318584.261845:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318584.261860:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 603 00000100:00100000:1.0:1767318584.261866:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982597632:12345-0@lo:400:kworker.0 00010000:00080000:0.0:1767318584.261871:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963b80 x1853163982597888/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318584.261883:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982597888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318584.261903:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982597632:12345-0@lo:400:kworker.0 Request processed in 37us (301us total) trans 0 rc 0/0 00010000:00080000:0.0:1767318584.261909:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631902d80 x1853163982597632/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318584.261910:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 603 00000100:00100000:0.0:1767318584.261917:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982597632:0@lo:400:kworker.0 00000100:00100000:2.0:1767318584.261988:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982597760:0@lo:38:kworker.0 00000100:00100000:2.0:1767318584.262023:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318584.262057:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982598016:0@lo:38:kworker.0 00000100:00100000:2.0:1767318584.262077:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318584.262098:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982598144:0@lo:38:kworker.0 00000100:00100000:2.0:1767318584.262123:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318584.262148:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982598272:0@lo:38:kworker.0 00000100:00100000:2.0:1767318584.262172:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318584.262200:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598144 00000100:00100000:2.0:1767318584.262204:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598144 00000100:00100000:2.0:1767318584.262226:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 221 00000100:00100000:2.0:1767318584.262231:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982598144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318584.262250:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318584.262364:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982597760 00000100:00100000:3.0:1767318584.262369:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982597760 00000100:00100000:3.0:1767318584.262388:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598016 00000100:00100000:3.0:1767318584.262391:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598016 00000100:00100000:3.0:1767318584.262403:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598272 00000100:00100000:3.0:1767318584.262405:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598272 00000100:00100000:3.0:1767318584.262415:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 782 00000100:00100000:3.0:1767318584.262419:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982597760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318584.262429:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318584.262459:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982597760:12345-0@lo:38:kworker.0 Request processed in 39us (435us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318584.262465:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 782 00000100:00100000:3.0:1767318584.262477:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 783 00000100:00100000:3.0:1767318584.262481:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982598016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318584.262490:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318584.262512:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982598016:12345-0@lo:38:kworker.0 Request processed in 30us (434us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318584.262521:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 783 00000100:00100000:1.0:1767318584.262531:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 784 00000100:00100000:1.0:1767318584.262535:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318584.262545:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318584.262572:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598272:12345-0@lo:38:kworker.0 Request processed in 35us (399us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318584.262577:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 784 00000100:00080000:3.0:1767318584.262579:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318584.262583:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318584.262587:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982597760:0@lo:38:kworker.0 00000100:00080000:3.0:1767318584.262595:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318584.262598:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318584.262600:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982598016:0@lo:38:kworker.0 00000100:00080000:3.0:1767318584.262608:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318584.262611:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318584.262614:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982598272:0@lo:38:kworker.0 00000100:00100000:2.0:1767318584.263145:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982598144:12345-0@lo:38:kworker.0 Request processed in 910us (1019us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318584.263153:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 221 00000100:00080000:3.0:1767318584.263201:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318584.263205:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318584.263208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982598144:0@lo:38:kworker.0 00010000:00010000:3.0:1767318584.388562:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318585.232466:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318585.232695:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915576704 offset 0 length 520: 4 00000100:00100000:1.0:1767318585.232725:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318585.233015:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576960 00000100:00100000:0.0:1767318585.233030:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576960 00000100:00080000:1.0:1767318585.233174:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318585.233225:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915576832 00000100:00100000:2.0:1767318585.233230:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915576832 00000100:00100000:2.0:1767318585.233295:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 785 00000100:00100000:0.0:1767318585.233358:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 604 00000100:00100000:0.0:1767318585.233368:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915576960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318585.233456:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915576960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (1002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318585.233465:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 604 00000100:00100000:2.0:1767318585.233735:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915576832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318585.233756:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318585.233791:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915576832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (1065us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318585.233797:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 785 00010000:00010000:1.0:1767318585.412711:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318586.437098:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318587.460661:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318588.358096:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318563/real 1767318563] req@ffff93463a8e2a00 x1853163982593920/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318588 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318588.358134:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e2a00 x1853163982593920/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318588 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318588.358157:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318588.358185:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318588.358190:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982593920:0@lo:250:kworker.0 00000100:00080000:2.0:1767318588.358309:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318588.358315:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318588.358319:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318588.358324:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3812 00000100:00080000:2.0:1767318588.358328:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318588.358335:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318588.358371:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56680 x1853163982598400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318588.358494:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982598528:0@lo:400:kworker.0 00000100:00100000:0.0:1767318588.358563:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318588.358683:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598528 00000100:00100000:3.0:1767318588.358684:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982598400:0@lo:250:kworker.0 00000100:00100000:1.0:1767318588.358693:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598528 00000400:00000100:3.0:1767318588.358715:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982598400 offset 0 length 520: 4 00000100:00100000:1.0:1767318588.358758:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 605 00000100:00100000:1.0:1767318588.358768:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982598528:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318588.358776:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318588.358837:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318588.358842:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318588.358844:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318588.358848:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3832 00000100:00080000:2.0:1767318588.358850:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318588.358854:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318588.358870:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604376300 x1853163982598656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318588.358927:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318588.358930:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318588.358932:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318588.358936:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3832 00000100:00080000:2.0:1767318588.358938:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318588.358943:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318588.358953:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982598912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318588.358962:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318588.358965:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318588.358967:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767318588.358969:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982598656:0@lo:38:kworker.0 00000100:00080000:2.0:1767318588.358971:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3832 00000100:00080000:2.0:1767318588.358973:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318588.358976:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318588.358988:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982599040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318588.359126:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359143:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982598912:0@lo:38:kworker.0 00000100:00100000:3.0:1767318588.359162:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359181:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982599040:0@lo:38:kworker.0 00000100:00100000:3.0:1767318588.359206:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359232:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598656 00000100:00100000:3.0:1767318588.359237:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598656 00000100:00100000:3.0:1767318588.359260:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598912 00000100:00100000:3.0:1767318588.359263:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598912 00000100:00100000:3.0:1767318588.359306:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 786 00000100:00100000:3.0:1767318588.359311:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598656:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318588.359333:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318588.359368:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598656:12345-0@lo:38:kworker.0 Request processed in 56us (241us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318588.359375:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 786 00000100:00100000:3.0:1767318588.359389:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 787 00000100:00100000:3.0:1767318588.359393:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318588.359402:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318588.359424:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982598912:12345-0@lo:38:kworker.0 Request processed in 29us (261us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318588.359429:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 787 00000100:00080000:3.0:1767318588.359463:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318588.359467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318588.359470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982598656:0@lo:38:kworker.0 00000100:00080000:3.0:1767318588.359477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318588.359480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318588.359483:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982598912:0@lo:38:kworker.0 00000100:00100000:3.0:1767318588.359518:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982598784:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359540:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359558:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318588.359564:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982599168:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359586:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359610:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599168 00000100:00100000:3.0:1767318588.359614:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599168 00000100:00100000:3.0:1767318588.359656:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 395 00000100:00100000:3.0:1767318588.359662:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982599168:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359697:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982599168:12345-0@lo:400:kworker.0 Request processed in 35us (110us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318588.359702:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 395 00000100:00080000:3.0:1767318588.359717:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318588.359725:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982599296:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359748:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.359779:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982599168:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359794:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599296 00000100:00100000:3.0:1767318588.359797:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599296 00000100:00100000:3.0:1767318588.359824:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 396 00000100:00100000:3.0:1767318588.359829:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982599296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359861:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982599296:12345-0@lo:400:kworker.0 Request processed in 32us (112us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318588.359867:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 396 00000100:00080000:3.0:1767318588.359881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318588.359896:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982599296:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.359912:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982598784 00000100:00100000:3.0:1767318588.359915:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982598784 00000100:00100000:3.0:1767318588.359942:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 606 00000100:00100000:3.0:1767318588.359949:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982598784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.362159:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982598784:12345-0@lo:400:kworker.0 Request processed in 2197us (2604us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318588.362178:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 606 00000100:00080000:3.0:1767318588.362234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767318588.362296:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320380 x1853163982598784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318604 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318588.362311:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982598784:0@lo:400:kworker.0 00000100:00100000:3.0:1767318588.362382:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599040 00000100:00100000:3.0:1767318588.362390:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599040 00000100:00100000:3.0:1767318588.362422:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 222 00000100:00100000:3.0:1767318588.362428:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982599040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318588.363168:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318588.363244:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982599040:12345-0@lo:38:kworker.0 Request processed in 814us (4035us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318588.363253:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 222 00000100:00080000:3.0:1767318588.363331:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318588.363336:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318588.363340:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982599040:0@lo:38:kworker.0 00000100:00080000:2.0:1767318588.365920:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318588.365931:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318588.365935:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318588.365943:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3832 00000100:00100000:1.0:1767318588.365965:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982598528:12345-0@lo:400:kworker.0 Request processed in 7193us (7395us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318588.365978:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 605 00000100:00080000:2.0:1767318588.366152:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318588.366158:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318588.366218:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320e00 x1853163982599424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318588.369024:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982599424:0@lo:38:kworker.0 00000100:00100000:3.0:1767318588.369148:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318588.369300:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599424 00000100:00100000:1.0:1767318588.369310:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599424 00010000:00080000:0.0:1767318588.369328:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604377b80 x1853163982598528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318604 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318588.369346:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982598528:0@lo:400:kworker.0 00000100:00100000:1.0:1767318588.369361:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 788 00000100:00100000:1.0:1767318588.369370:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982599424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318588.369395:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318588.369445:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982599424:12345-0@lo:38:kworker.0 Request processed in 74us (296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318588.369452:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 788 00000100:00080000:2.0:1767318588.369566:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318588.369573:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318588.369576:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982599424:0@lo:38:kworker.0 00010000:00010000:1.0:1767318588.488531:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318588.868602:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982599552:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318588.868685:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318588.868693:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318588.868702:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982599680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318588.869157:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599552 00000100:00100000:0.0:1767318588.869168:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599552 00000100:00100000:0.0:1767318588.869232:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 426 00000100:00100000:0.0:1767318588.869244:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982599552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318588.869302:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318588.869562:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318588.870224:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599680 00000100:00100000:3.0:1767318588.870232:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599680 00000100:00100000:3.0:1767318588.870494:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 427 00000100:00100000:3.0:1767318588.870504:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982599680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318588.870531:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318588.870609:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982599680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (1047us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318588.870626:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 427 00000100:00100000:2.0:1767318588.870726:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982599680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318588.871877:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982599552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2630us (3178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318588.871891:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 426 00000100:00100000:1.0:1767318588.871981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982599552:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318589.508509:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318590.350641:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318590.350691:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318590.350866:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577216 00000100:00100000:2.0:1767318590.350878:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577216 00000100:00080000:0.0:1767318590.351057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318590.351265:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577088 00000100:00100000:2.0:1767318590.351302:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577088 00000100:00100000:1.0:1767318590.356319:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 607 00000100:00100000:1.0:1767318590.356353:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915577216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318590.356466:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915577216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (5836us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318590.356474:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 607 00000100:00100000:1.0:1767318590.356504:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 789 00000100:00100000:1.0:1767318590.356508:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346309f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577088:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318590.356529:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318590.356557:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346309f6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (5866us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318590.356562:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 789 00010000:00010000:3.0:1767318590.532515:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318591.557706:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318592.580891:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318593.476766:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318593.476776:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318593.476861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982599808:0@lo:400:kworker.0 00000100:00100000:2.0:1767318593.476980:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318593.477102:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318593.477106:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318593.477109:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318593.477115:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3836 00000100:00080000:3.0:1767318593.477119:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318593.477125:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318593.477149:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163982599936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318593.477185:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318593.477188:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318593.477191:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318593.477194:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3836 00000100:00080000:3.0:1767318593.477197:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318593.477201:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318593.477243:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1500 x1853163982600192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318593.477253:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318593.477256:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318593.477300:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318593.477305:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3836 00000100:00080000:3.0:1767318593.477308:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318593.477312:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318593.477326:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0700 x1853163982600320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318593.477381:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599808 00000100:00100000:1.0:1767318593.477389:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599808 00000100:00100000:0.0:1767318593.477401:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982600576:0@lo:400:kworker.0 00000100:00100000:1.0:1767318593.477444:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 608 00000100:00100000:0.0:1767318593.477446:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318593.477454:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982599808:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318593.477475:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318593.477506:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982599936:0@lo:38:kworker.0 00000100:00100000:0.0:1767318593.477542:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318593.477561:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982600192:0@lo:38:kworker.0 00000100:00100000:0.0:1767318593.477580:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318593.477596:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982600320:0@lo:38:kworker.0 00000100:00100000:0.0:1767318593.477617:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318593.477645:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982599936 00000100:00100000:0.0:1767318593.477649:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982599936 00000100:00100000:0.0:1767318593.477684:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600192 00000100:00100000:0.0:1767318593.477686:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600192 00000100:00100000:0.0:1767318593.477696:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 790 00000100:00100000:0.0:1767318593.477700:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982599936:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318593.477716:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318593.477745:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982599936:12345-0@lo:38:kworker.0 Request processed in 44us (202us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318593.477751:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 790 00000100:00100000:0.0:1767318593.478047:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 791 00000100:00100000:0.0:1767318593.478052:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982600192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318593.478064:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318593.478112:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982600192:12345-0@lo:38:kworker.0 Request processed in 58us (531us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318593.478117:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 791 00000100:00100000:0.0:1767318593.478134:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982600064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478155:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318593.478174:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318593.478179:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982600448:0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478199:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318593.478254:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318593.478408:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318593.478416:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982599936:0@lo:38:kworker.0 00000100:00080000:0.0:1767318593.478428:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318593.478431:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318593.478433:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982600192:0@lo:38:kworker.0 00000100:00100000:0.0:1767318593.478461:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600576 00000100:00100000:0.0:1767318593.478464:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600576 00000100:00100000:0.0:1767318593.478505:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600448 00000100:00100000:0.0:1767318593.478508:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600448 00000100:00100000:0.0:1767318593.478661:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 397 00000100:00100000:0.0:1767318593.478672:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982600576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478710:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982600576:12345-0@lo:400:kworker.0 Request processed in 40us (1275us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318593.478716:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 397 00000100:00100000:0.0:1767318593.478727:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 398 00000100:00100000:0.0:1767318593.478733:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982600448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478768:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982600448:12345-0@lo:400:kworker.0 Request processed in 35us (568us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318593.478773:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 398 00000100:00080000:0.0:1767318593.478794:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318593.478796:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318593.478804:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600064 00000100:00100000:0.0:1767318593.478807:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600064 00000100:00100000:0.0:1767318593.478829:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 609 00000100:00100000:0.0:1767318593.478835:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982600064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478863:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982600064:12345-0@lo:400:kworker.0 Request processed in 28us (706us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318593.478868:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 609 00000100:00080000:0.0:1767318593.478881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318593.478902:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982600576:0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.478923:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982600448:0@lo:400:kworker.0 00010000:00080000:0.0:1767318593.478933:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a913800 x1853163982600064/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318609 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318593.478943:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982600064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318593.479013:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600320 00000100:00100000:0.0:1767318593.479017:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600320 00000100:00100000:0.0:1767318593.479036:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 223 00000100:00100000:0.0:1767318593.479040:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982600320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318593.479052:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318593.479082:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982600320:12345-0@lo:38:kworker.0 Request processed in 39us (1464us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318593.479091:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 223 00000100:00080000:3.0:1767318593.480058:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318593.480066:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318593.480069:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318593.480075:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3836 00000100:00080000:3.0:1767318593.480080:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318593.480086:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318593.480124:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163982600704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318593.480438:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982600704:0@lo:38:kworker.0 00000100:00100000:3.0:1767318593.480491:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318593.480549:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318593.480554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318593.480557:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982600320:0@lo:38:kworker.0 00000100:00100000:3.0:1767318593.480603:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600704 00000100:00100000:3.0:1767318593.480608:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600704 00000100:00100000:3.0:1767318593.480638:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 792 00000100:00100000:3.0:1767318593.480644:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982600704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318593.480662:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318593.480705:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982600704:12345-0@lo:38:kworker.0 Request processed in 60us (213us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318593.480710:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 792 00000100:00080000:3.0:1767318593.480734:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318593.480738:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318593.480741:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982600704:0@lo:38:kworker.0 00010000:00080000:2.0:1767318593.481331:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6a00 x1853163982599808/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318609 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318593.481344:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982599808:0@lo:400:kworker.0 00000100:00100000:1.0:1767318593.481452:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982599808:12345-0@lo:400:kworker.0 Request processed in 3999us (4470us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318593.481461:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 608 00010000:00010000:3.0:1767318593.606262:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318593.988550:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318593.988582:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982600832:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318593.988670:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318593.988822:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600832 00000100:00100000:1.0:1767318593.988830:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600832 00000100:00100000:1.0:1767318593.988892:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 428 00000100:00100000:1.0:1767318593.988902:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982600832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318593.988938:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318593.988987:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982600832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318593.988994:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 428 00000100:00100000:0.0:1767318593.989071:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982600832:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318593.990707:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318593.990725:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982600960:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318593.990778:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318593.991117:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982600960 00000100:00100000:0.0:1767318593.991124:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982600960 00000100:00100000:0.0:1767318593.991183:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 429 00000100:00100000:0.0:1767318593.991192:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982600960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318593.991221:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318593.991303:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982600960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (523us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318593.991311:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 429 00000100:00100000:3.0:1767318593.991397:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982600960:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318594.628562:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318595.468197:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318595.468331:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577472 00000100:00100000:2.0:1767318595.468341:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318595.468342:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577472 00000100:00080000:3.0:1767318595.468394:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318595.470170:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577344 00000100:00100000:0.0:1767318595.470184:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577344 00000100:00100000:0.0:1767318595.470264:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 610 00000100:00100000:0.0:1767318595.470295:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915577472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318595.470317:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 793 00000100:00100000:1.0:1767318595.470327:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318595.470350:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318595.470588:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915577472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 313us (2401us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318595.470598:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 610 00000100:00100000:1.0:1767318595.473798:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3458us (5444us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318595.473816:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 793 00010000:00010000:1.0:1767318595.652558:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318596.676588:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318597.700520:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318598.596880:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318598.596890:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318598.596955:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318598.596960:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318598.596964:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318598.596974:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3842 00000100:00080000:2.0:1767318598.596980:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767318598.596984:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767318598.596987:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318598.597009:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982601088:0@lo:400:kworker.0 00000100:00100000:2.0:1767318598.597014:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321880 x1853163982601216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318598.597048:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318598.597051:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318598.597053:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318598.597057:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3842 00000100:00080000:2.0:1767318598.597059:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318598.597063:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318598.597074:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323480 x1853163982601472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318598.597083:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318598.597087:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318598.597089:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318598.597092:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3842 00000100:00080000:2.0:1767318598.597095:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318598.597098:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318598.597337:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982601344:0@lo:400:kworker.0 00000100:00100000:3.0:1767318598.602987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982601216:0@lo:38:kworker.0 00000100:00100000:2.0:1767318598.603094:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321500 x1853163982601600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318598.603149:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318598.603153:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318598.603157:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318598.603162:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3842 00000100:00080000:2.0:1767318598.603166:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318598.603172:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318598.603182:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318598.603191:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982601856:0@lo:400:kworker.0 00000100:00100000:0.0:1767318598.607223:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318598.607304:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318598.607317:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982601728:0@lo:400:kworker.0 00000100:00100000:0.0:1767318598.607353:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318598.607392:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601728 00000100:00100000:0.0:1767318598.607399:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601728 00000100:00100000:1.0:1767318598.609755:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601344 00000100:00100000:1.0:1767318598.609767:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601344 00000100:00100000:1.0:1767318598.609827:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 611 00000100:00100000:1.0:1767318598.609839:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982601344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318598.610798:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318598.610894:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 399 00000100:00100000:0.0:1767318598.610905:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982601728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318598.612841:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982601344:12345-0@lo:400:kworker.0 Request processed in 2996us (5619us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318598.613371:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 611 00000100:00100000:1.0:1767318598.613407:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601216 00000100:00100000:1.0:1767318598.613416:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601216 00000100:00100000:1.0:1767318598.613470:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 794 00000100:00100000:1.0:1767318598.613475:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982601216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318598.613497:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318598.613556:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982601216:12345-0@lo:38:kworker.0 Request processed in 79us (2760us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318598.613563:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 794 00000100:00100000:1.0:1767318598.613612:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318598.613641:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601088 00000100:00100000:1.0:1767318598.613644:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601088 00000100:00100000:1.0:1767318598.613673:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 612 00000100:00100000:1.0:1767318598.613679:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982601088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318598.613722:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982601088:12345-0@lo:400:kworker.0 Request processed in 44us (111us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318598.613728:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 612 00010000:00080000:1.0:1767318598.613766:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322d80 x1853163982601088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318614 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318598.613779:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982601088:0@lo:400:kworker.0 00000100:00080000:3.0:1767318598.613867:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318598.613875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318598.613879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982601216:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.613890:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982601472:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.613922:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318598.613953:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982601600:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.613983:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318598.614021:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601472 00000100:00100000:3.0:1767318598.614025:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601472 00000100:00100000:3.0:1767318598.614045:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795 00000100:00100000:3.0:1767318598.614050:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982601472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318598.614061:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318598.614088:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982601472:12345-0@lo:38:kworker.0 Request processed in 36us (165us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318598.614093:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795 00000100:00080000:3.0:1767318598.614117:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318598.614121:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318598.614124:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982601472:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.614176:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1c00 x1853163982601984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318598.614204:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982601984:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.614225:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318598.614245:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601984 00000100:00100000:3.0:1767318598.614248:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601984 00000100:00100000:3.0:1767318598.614265:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 796 00000100:00100000:3.0:1767318598.614287:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982601984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318598.614298:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318598.614325:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982601984:12345-0@lo:38:kworker.0 Request processed in 56us (100us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318598.614331:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 796 00000100:00080000:3.0:1767318598.614350:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318598.614354:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318598.614358:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982601984:0@lo:38:kworker.0 00000100:00100000:3.0:1767318598.614401:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601600 00000100:00100000:3.0:1767318598.614404:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601600 00000100:00100000:3.0:1767318598.614437:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 224 00000100:00100000:3.0:1767318598.614441:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982601600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318598.614451:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318598.614856:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318598.614891:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982601856 00000100:00100000:2.0:1767318598.614895:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982601856 00000100:00100000:2.0:1767318598.614929:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 400 00000100:00100000:2.0:1767318598.614936:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982601856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318598.614972:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982601856:12345-0@lo:400:kworker.0 Request processed in 37us (117us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318598.614977:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 400 00000100:00100000:2.0:1767318598.615003:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982601856:0@lo:400:kworker.0 00000100:00080000:2.0:1767318598.615032:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318598.615037:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318598.615039:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318598.615155:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982601728:12345-0@lo:400:kworker.0 Request processed in 4252us (7800us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318598.615163:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 399 00000100:00100000:0.0:1767318598.615190:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982601728:0@lo:400:kworker.0 00010000:00080000:0.0:1767318598.615202:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322300 x1853163982601344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318614 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318598.615210:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982601344:0@lo:400:kworker.0 00000100:00100000:2.0:1767318598.615782:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982601600:12345-0@lo:38:kworker.0 Request processed in 1336us (1794us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318598.615792:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 224 00000100:00080000:3.0:1767318598.615846:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318598.615850:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318598.615854:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982601600:0@lo:38:kworker.0 00010000:00010000:1.0:1767318598.730696:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318599.108733:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318599.108772:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318599.108800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982602240:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318599.108935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318599.108937:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982602112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318599.109134:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602240 00000100:00100000:1.0:1767318599.109143:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602240 00000100:00100000:1.0:1767318599.109210:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 430 00000100:00100000:1.0:1767318599.109221:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982602240:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318599.109257:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318599.113241:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318599.113369:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602112 00000100:00100000:2.0:1767318599.113379:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602112 00000100:00100000:2.0:1767318599.113447:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 431 00000100:00100000:2.0:1767318599.113457:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982602112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318599.113497:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318599.113555:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982602112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (326us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318599.113563:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 431 00000100:00100000:2.0:1767318599.113616:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982602112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318599.131032:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982602240:12345-0@lo:13:osp-pre-0-1.0 Request processed in 21798us (22093us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318599.131871:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982602240:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318599.132616:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 430 00010000:00010000:3.0:1767318599.749209:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318600.588697:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318600.588750:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318600.588836:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577600 00000100:00100000:1.0:1767318600.588847:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577600 00000100:00100000:1.0:1767318600.588895:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577728 00000100:00100000:1.0:1767318600.588898:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577728 00000100:00100000:1.0:1767318600.588962:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 613 00000100:00100000:1.0:1767318600.588972:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915577728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318600.589051:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915577728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318600.589058:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 613 00000100:00100000:1.0:1767318600.589076:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 797 00000100:00100000:1.0:1767318600.589081:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318600.589101:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318600.589126:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (443us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318600.589132:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 797 00000100:00080000:3.0:1767318600.589742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318600.777034:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318601.796573:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318602.822300:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318603.716460:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318603.716482:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318603.716563:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318603.716568:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318603.716573:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318603.716585:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3847 00000100:00080000:2.0:1767318603.716591:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318603.716601:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318603.716634:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321500 x1853163982602496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318603.716693:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318603.716697:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318603.716699:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318603.716703:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3847 00000100:00080000:2.0:1767318603.716705:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318603.716710:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318603.716720:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163982602752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318603.716727:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318603.716730:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318603.716732:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318603.716734:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3847 00000100:00080000:2.0:1767318603.716737:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318603.716740:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318603.716763:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323480 x1853163982602880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318603.716783:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318603.716786:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318603.716788:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318603.716792:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3847 00000100:00080000:2.0:1767318603.716794:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318603.716798:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318603.716806:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982603264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318603.716835:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982602624:0@lo:400:kworker.0 00000100:00100000:2.0:1767318603.716925:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318603.717004:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318603.717011:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982603008:0@lo:400:kworker.0 00000100:00100000:2.0:1767318603.717038:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318603.717043:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982603136:0@lo:400:kworker.0 00000100:00100000:0.0:1767318603.717072:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318603.717092:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603008 00000100:00100000:0.0:1767318603.717093:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318603.717098:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982602368:0@lo:400:kworker.0 00000100:00100000:2.0:1767318603.717101:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603008 00000100:00100000:0.0:1767318603.717119:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318603.717147:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602624 00000100:00100000:0.0:1767318603.717151:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602624 00000100:00100000:2.0:1767318603.717157:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603136 00000100:00100000:2.0:1767318603.717159:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603136 00000100:00100000:0.0:1767318603.717178:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602368 00000100:00100000:0.0:1767318603.717180:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602368 00000100:00100000:2.0:1767318603.717183:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 401 00000100:00100000:2.0:1767318603.717191:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982603008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318603.717200:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 614 00000100:00100000:0.0:1767318603.717205:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982602624:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318603.717247:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982602624:12345-0@lo:400:kworker.0 Request processed in 41us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318603.717247:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982603008:12345-0@lo:400:kworker.0 Request processed in 56us (207us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318603.717257:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 614 00000100:00100000:2.0:1767318603.717257:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 401 00000100:00100000:0.0:1767318603.717295:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 615 00000100:00100000:2.0:1767318603.717295:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 402 00000100:00100000:0.0:1767318603.717302:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982602368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318603.717302:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982603136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318603.717343:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982602368:12345-0@lo:400:kworker.0 Request processed in 41us (222us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318603.717343:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982603136:12345-0@lo:400:kworker.0 Request processed in 41us (270us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318603.717350:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 615 00000100:00100000:2.0:1767318603.717350:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 402 00010000:00080000:0.0:1767318603.717389:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321f80 x1853163982602368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318619 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318603.717398:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982602368:0@lo:400:kworker.0 00000100:00100000:2.0:1767318603.717398:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982603008:0@lo:400:kworker.0 00000100:00100000:0.0:1767318603.717408:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982603136:0@lo:400:kworker.0 00010000:00080000:2.0:1767318603.717408:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322300 x1853163982602624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318619 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318603.717416:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982602624:0@lo:400:kworker.0 00000100:00100000:1.0:1767318603.717584:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982602496:0@lo:38:kworker.0 00000100:00100000:1.0:1767318603.717617:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318603.717654:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982602752:0@lo:38:kworker.0 00000100:00100000:1.0:1767318603.717738:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318603.717763:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982602880:0@lo:38:kworker.0 00000100:00080000:3.0:1767318603.717766:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318603.717772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318603.717774:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318603.717786:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602496 00000100:00100000:1.0:1767318603.717788:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318603.717790:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602496 00000100:00100000:1.0:1767318603.717806:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982603264:0@lo:38:kworker.0 00000100:00100000:3.0:1767318603.717809:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602752 00000100:00100000:3.0:1767318603.717811:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602752 00000100:00100000:3.0:1767318603.717824:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798 00000100:00100000:1.0:1767318603.717825:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318603.717830:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982602496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318603.717847:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318603.717863:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603264 00000100:00100000:1.0:1767318603.717866:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603264 00000100:00100000:1.0:1767318603.717880:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 799 00000100:00100000:3.0:1767318603.717883:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982602496:12345-0@lo:38:kworker.0 Request processed in 53us (267us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318603.717884:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982602752:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318603.717889:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798 00010000:00020000:1.0:1767318603.717894:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318603.717902:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800 00000100:00100000:3.0:1767318603.717905:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982603264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318603.717913:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318603.717916:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982602752:12345-0@lo:38:kworker.0 Request processed in 31us (179us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318603.717922:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 799 00000100:00100000:3.0:1767318603.717930:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982603264:12345-0@lo:38:kworker.0 Request processed in 24us (105us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318603.717935:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800 00000100:00080000:1.0:1767318603.717944:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318603.717948:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318603.717951:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982602496:0@lo:38:kworker.0 00000100:00080000:1.0:1767318603.717960:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318603.717963:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318603.717965:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982602752:0@lo:38:kworker.0 00000100:00080000:1.0:1767318603.717972:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318603.717975:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318603.717978:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982603264:0@lo:38:kworker.0 00000100:00100000:2.0:1767318603.719165:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982602880 00000100:00100000:2.0:1767318603.719176:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982602880 00000100:00100000:2.0:1767318603.719229:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 225 00000100:00100000:2.0:1767318603.719237:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982602880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318603.719250:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318603.719360:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982602880:12345-0@lo:38:kworker.0 Request processed in 122us (1569us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318603.719367:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 225 00000100:00080000:1.0:1767318603.720142:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318603.720151:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318603.720155:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982602880:0@lo:38:kworker.0 00010000:00010000:1.0:1767318603.853082:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318604.228900:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318604.228928:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982603392:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318604.229006:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318604.229132:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603392 00000100:00100000:0.0:1767318604.229140:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603392 00000100:00100000:0.0:1767318604.229193:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 432 00000100:00100000:0.0:1767318604.229203:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982603392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318604.229237:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318604.229337:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982603392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318604.229344:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 432 00000100:00100000:3.0:1767318604.229487:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982603392:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318604.229954:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982603520:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318604.229987:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318604.235220:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603520 00000100:00100000:1.0:1767318604.235241:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603520 00000100:00100000:1.0:1767318604.235348:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 433 00000100:00100000:1.0:1767318604.235360:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982603520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318604.235409:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318604.236049:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982603520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 689us (6059us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318604.236059:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 433 00000100:00100000:2.0:1767318604.237378:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982603520:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318604.868602:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318605.707712:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318605.707947:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577984 00000100:00100000:2.0:1767318605.707957:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577984 00000100:00100000:2.0:1767318605.708030:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 616 00000100:00100000:2.0:1767318605.708039:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915577984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318605.708127:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915577984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (425us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318605.708135:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 616 00000100:00080000:3.0:1767318605.708924:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318605.709725:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318605.709943:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915577856 00000100:00100000:0.0:1767318605.709951:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915577856 00000100:00100000:0.0:1767318605.709993:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 801 00000100:00100000:0.0:1767318605.710000:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318605.710022:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318605.710080:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915577856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 79us (359us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318605.710088:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 801 00010000:00010000:3.0:1767318605.893367:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318606.917062:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318607.941856:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318608.836560:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318608.836573:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318608.836643:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318608.836652:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318608.836657:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318608.836665:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3852 00000100:00080000:2.0:1767318608.836669:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318608.836675:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318608.836700:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982603776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318608.836735:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318608.836738:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318608.836740:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318608.836744:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3852 00000100:00080000:2.0:1767318608.836746:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318608.836750:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318608.836762:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982604032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318608.836771:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318608.836774:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318608.836776:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318608.836779:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3852 00000100:00080000:2.0:1767318608.836781:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318608.836784:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318608.836795:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163982604160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318608.836815:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318608.836818:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318608.836820:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318608.836824:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3852 00000100:00080000:2.0:1767318608.836826:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318608.836830:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318608.836849:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982604544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318608.837140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982604288:0@lo:400:kworker.0 00000100:00100000:2.0:1767318608.837206:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318608.837264:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982603776:0@lo:38:kworker.0 00000100:00100000:2.0:1767318608.837282:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318608.837288:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982603904:0@lo:400:kworker.0 00000100:00100000:1.0:1767318608.837328:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604288 00000100:00100000:2.0:1767318608.837328:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318608.837335:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604288 00000100:00100000:2.0:1767318608.837360:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603904 00000100:00100000:2.0:1767318608.837364:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603904 00000100:00100000:1.0:1767318608.837384:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 403 00000100:00100000:2.0:1767318608.837390:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 617 00000100:00100000:1.0:1767318608.837392:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982604288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318608.837395:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982603904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318608.837449:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982604288:12345-0@lo:400:kworker.0 Request processed in 56us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318608.837456:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 403 00000100:00080000:1.0:1767318608.837479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318608.837482:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982603904:12345-0@lo:400:kworker.0 Request processed in 86us (154us total) trans 0 rc 0/0 00000100:00080000:1.0:1767318608.837484:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318608.837488:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 617 00000100:00100000:1.0:1767318608.837492:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318608.837520:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982604032:0@lo:38:kworker.0 00000100:00100000:0.0:1767318608.837540:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982604416:0@lo:400:kworker.0 00000100:00100000:1.0:1767318608.837541:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318608.837596:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982603648:0@lo:400:kworker.0 00000100:00100000:0.0:1767318608.837686:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318608.837799:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603776 00000100:00100000:2.0:1767318608.837803:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603776 00000100:00100000:2.0:1767318608.837824:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 802 00000100:00100000:2.0:1767318608.837828:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982603776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318608.838921:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318608.838977:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982603776:12345-0@lo:38:kworker.0 Request processed in 1146us (1482us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318608.838986:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 802 00010000:00080000:2.0:1767318608.839195:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323480 x1853163982603904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318624 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318608.839210:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982603904:0@lo:400:kworker.0 00000100:00100000:2.0:1767318608.839222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982604288:0@lo:400:kworker.0 00000100:00100000:2.0:1767318608.839300:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604416 00000100:00100000:2.0:1767318608.839305:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604416 00000100:00100000:2.0:1767318608.839351:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 404 00000100:00100000:2.0:1767318608.839359:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982604416:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318608.839403:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982604416:12345-0@lo:400:kworker.0 Request processed in 45us (1717us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318608.839410:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 404 00000100:00080000:2.0:1767318608.839447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318608.839478:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982604416:0@lo:400:kworker.0 00000100:00100000:1.0:1767318608.843373:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982604160:0@lo:38:kworker.0 00000100:00100000:1.0:1767318608.843443:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318608.843474:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982604544:0@lo:38:kworker.0 00000100:00100000:1.0:1767318608.843499:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318608.843548:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318608.843554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318608.843569:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982603776:0@lo:38:kworker.0 00000100:00100000:1.0:1767318608.843607:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318608.843643:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604032 00000100:00100000:1.0:1767318608.843649:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604032 00000100:00100000:1.0:1767318608.843674:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604544 00000100:00100000:1.0:1767318608.843688:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604544 00000100:00100000:1.0:1767318608.843697:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982603648 00000100:00100000:1.0:1767318608.843700:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982603648 00000100:00100000:1.0:1767318608.843735:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 618 00000100:00100000:1.0:1767318608.843743:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982603648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318608.844120:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604160 00000100:00100000:3.0:1767318608.844126:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604160 00000100:00100000:3.0:1767318608.844149:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 226 00000100:00100000:1.0:1767318608.844151:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982603648:12345-0@lo:400:kworker.0 Request processed in 408us (542us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318608.844154:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982604160:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318608.844158:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 618 00010000:00020000:3.0:1767318608.844172:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318608.844175:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 803 00000100:00100000:1.0:1767318608.844179:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982604032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318608.844187:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318608.844209:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982604032:12345-0@lo:38:kworker.0 Request processed in 28us (6666us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318608.844210:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982604160:12345-0@lo:38:kworker.0 Request processed in 54us (766us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318608.844216:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 803 00000100:00100000:3.0:1767318608.844216:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 226 00000100:00100000:1.0:1767318608.844237:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804 00000100:00100000:1.0:1767318608.844241:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982604544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318608.844249:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318608.844266:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318608.844285:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982604544:12345-0@lo:38:kworker.0 Request processed in 28us (771us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318608.844290:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804 00010000:00080000:1.0:1767318608.844318:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321c00 x1853163982603648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318624 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318608.844330:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982603648:0@lo:400:kworker.0 00000100:00080000:1.0:1767318608.844353:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318608.844357:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318608.844360:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982604032:0@lo:38:kworker.0 00000100:00080000:1.0:1767318608.844367:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318608.844370:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318608.844373:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982604160:0@lo:38:kworker.0 00000100:00080000:1.0:1767318608.844380:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318608.844383:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318608.844386:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982604544:0@lo:38:kworker.0 00010000:00010000:3.0:1767318608.964643:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318609.348677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982604672:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318609.348775:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318609.349257:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604672 00000100:00100000:3.0:1767318609.349267:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604672 00000100:00100000:3.0:1767318609.349345:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 434 00000100:00100000:3.0:1767318609.349356:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982604672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318609.349385:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318609.349636:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982604672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 282us (860us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318609.349644:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 434 00000100:00100000:1.0:1767318609.349967:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982604672:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318609.350117:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982604800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318609.350156:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318609.350246:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604800 00000100:00100000:1.0:1767318609.350251:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604800 00000100:00100000:1.0:1767318609.350321:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 435 00000100:00100000:1.0:1767318609.350329:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982604800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318609.350348:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318609.350383:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982604800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318609.350390:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 435 00000100:00100000:0.0:1767318609.350624:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982604800:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318609.809251:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318609.809357:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915578112 offset 0 length 520: 4 00000100:00100000:1.0:1767318609.809393:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318609.810218:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578368 00000100:00100000:0.0:1767318609.810230:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578368 00000100:00100000:0.0:1767318609.810372:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578240 00000100:00100000:0.0:1767318609.810376:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578240 00000100:00100000:0.0:1767318609.810403:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 619 00000100:00100000:0.0:1767318609.810414:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915578368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318609.810488:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915578368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (1375us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318609.810497:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 619 00000100:00100000:0.0:1767318609.810515:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 805 00000100:00100000:0.0:1767318609.810520:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915578240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318609.810541:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318609.810569:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915578240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (1175us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318609.810576:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 805 00000100:00080000:3.0:1767318609.812767:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318609.989012:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318611.012864:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318612.038974:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318613.060805:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318613.957876:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.957887:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318613.957890:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318588/real 1767318588] req@ffff934639e56680 x1853163982598400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318613 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318613.957950:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.957954:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318613.957958:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318613.957964:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3857 00000100:00080000:1.0:1767318613.957968:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318613.957975:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318613.957995:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4a80 x1853163982605056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318613.958006:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934639e56680 x1853163982598400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318613 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318613.958024:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.958028:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318613.958030:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318613.958033:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3857 00000100:00080000:1.0:1767318613.958036:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318613.958039:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318613.958040:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982605056:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.958050:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5180 x1853163982605312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318613.958058:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.958061:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318613.958063:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318613.958066:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3857 00000100:00080000:1.0:1767318613.958068:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318613.958071:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318613.958081:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982605440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318613.958103:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318613.958125:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.958161:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318613.958163:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318613.958167:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3857 00000100:00080000:1.0:1767318613.958169:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318613.958173:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:2.0:1767318613.958174:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767318613.958184:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163982605824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318613.958186:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318613.958190:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982598400:0@lo:250:kworker.0 00000100:00080000:1.0:1767318613.958197:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318613.958200:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318613.958202:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318613.958204:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3836 00000100:00080000:1.0:1767318613.958206:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318613.958210:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318613.958210:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982605312:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.958218:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583b80 x1853163982605952/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318613.958231:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.958241:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982605184:0@lo:400:kworker.0 00000100:00100000:2.0:1767318613.958247:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982605440:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.958268:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318613.958294:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318613.958316:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982605824:0@lo:38:kworker.0 00000100:00100000:1.0:1767318613.958319:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318613.958325:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982605568:0@lo:400:kworker.0 00000100:00100000:2.0:1767318613.958337:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318613.958344:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318613.958352:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982605952:0@lo:250:kworker.0 00000100:00100000:1.0:1767318613.958383:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605056 00000100:00100000:1.0:1767318613.958392:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605056 00000100:00100000:1.0:1767318613.958420:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605312 00000100:00100000:1.0:1767318613.958424:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605312 00000100:00100000:1.0:1767318613.958433:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605184 00000100:00100000:1.0:1767318613.958436:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605184 00000100:00100000:1.0:1767318613.958469:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605824 00000100:00100000:1.0:1767318613.958472:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605824 00000100:00100000:3.0:1767318613.958631:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982604928:0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.958667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318613.958685:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318613.958691:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982605696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.958714:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318613.958756:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605440 00000100:00100000:3.0:1767318613.958761:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605440 00000100:00100000:3.0:1767318613.958790:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 227 00000100:00100000:3.0:1767318613.958799:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982605440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318613.958823:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318613.958865:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982605440:12345-0@lo:38:kworker.0 Request processed in 66us (596us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318613.958875:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 227 00000100:00100000:3.0:1767318613.958895:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982604928 00000100:00100000:3.0:1767318613.958898:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982604928 00000100:00100000:3.0:1767318613.958915:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 620 00000100:00100000:3.0:1767318613.958921:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982605184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.958962:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982605184:12345-0@lo:400:kworker.0 Request processed in 42us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318613.958968:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 620 00000100:00100000:3.0:1767318613.958983:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 621 00000100:00100000:3.0:1767318613.958989:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982604928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.959019:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982604928:12345-0@lo:400:kworker.0 Request processed in 31us (352us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318613.959025:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 621 00000100:00100000:3.0:1767318613.959036:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806 00000100:00100000:3.0:1767318613.959040:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982605056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318613.959049:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318613.959072:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982605056:12345-0@lo:38:kworker.0 Request processed in 30us (968us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318613.959077:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806 00000100:00100000:3.0:1767318613.959089:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807 00000100:00100000:3.0:1767318613.959093:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982605312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318613.959101:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318613.959122:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982605312:12345-0@lo:38:kworker.0 Request processed in 28us (890us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318613.959138:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807 00000100:00100000:3.0:1767318613.959158:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605568 00000100:00100000:3.0:1767318613.959162:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605568 00000100:00100000:3.0:1767318613.959188:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982605696 00000100:00100000:3.0:1767318613.959191:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982605696 00000100:00100000:3.0:1767318613.959212:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 405 00000100:00100000:3.0:1767318613.959218:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982605568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.959248:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982605568:12345-0@lo:400:kworker.0 Request processed in 31us (903us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318613.959254:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 405 00000100:00100000:3.0:1767318613.959264:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 406 00000100:00100000:3.0:1767318613.959304:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982605696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.959332:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982605696:12345-0@lo:400:kworker.0 Request processed in 63us (617us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318613.959337:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 406 00000100:00100000:3.0:1767318613.959389:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982605568:0@lo:400:kworker.0 00010000:00080000:3.0:1767318613.959401:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7b80 x1853163982605184/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318629 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318613.959409:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982605184:0@lo:400:kworker.0 00000100:00100000:3.0:1767318613.959435:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982605696:0@lo:400:kworker.0 00010000:00080000:3.0:1767318613.959443:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4e00 x1853163982604928/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318629 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318613.959450:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982604928:0@lo:400:kworker.0 00000400:00000100:2.0:1767318613.960350:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982605952 offset 0 length 520: 4 00000100:00080000:2.0:1767318613.960409:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318613.960415:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318613.960419:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982605312:0@lo:38:kworker.0 00000100:00080000:2.0:1767318613.960429:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318613.960432:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318613.960435:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982605440:0@lo:38:kworker.0 00000100:00080000:2.0:1767318613.960443:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318613.960447:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318613.960449:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982605056:0@lo:38:kworker.0 00000100:00080000:2.0:1767318613.960982:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318613.960988:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318613.960991:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318613.961254:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 808 00000100:00100000:1.0:1767318613.961262:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982605824:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318613.961306:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318613.961372:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982605824:12345-0@lo:38:kworker.0 Request processed in 101us (3025us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318613.961380:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 808 00000100:00080000:2.0:1767318613.962310:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318613.962316:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318613.962321:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982605824:0@lo:38:kworker.0 00010000:00010000:1.0:1767318614.087311:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318614.469958:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982606080:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318614.470028:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318614.470134:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606080 00000100:00100000:3.0:1767318614.470142:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606080 00000100:00100000:3.0:1767318614.470213:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 436 00000100:00100000:3.0:1767318614.470223:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982606080:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318614.474036:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982606208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318614.474109:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318614.474179:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606208 00000100:00100000:0.0:1767318614.474188:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606208 00000100:00100000:0.0:1767318614.474237:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 437 00000100:00100000:0.0:1767318614.474246:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982606208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318614.474291:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318614.474334:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982606208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318614.474341:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 437 00000100:00100000:0.0:1767318614.474387:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982606208:0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318614.475218:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318614.475308:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982606080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5087us (5279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318614.475320:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 436 00000100:00100000:0.0:1767318614.475545:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982606080:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318614.928383:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318614.928482:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318614.935879:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578624 00000100:00100000:0.0:1767318614.935897:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578624 00000100:00100000:2.0:1767318614.936157:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578496 00000100:00100000:2.0:1767318614.936164:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578496 00000100:00100000:0.0:1767318614.936315:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 622 00000100:00100000:0.0:1767318614.936328:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915578624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318614.936429:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915578624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 104us (8055us total) trans 0 rc 0/0 00000100:00080000:3.0:1767318614.936476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318614.936551:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 809 00000100:00100000:2.0:1767318614.936558:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915578496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318614.936579:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318614.936627:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915578496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 67us (8144us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318614.936634:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 809 00000100:00100000:0.0:1767318614.941403:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 622 00010000:00010000:3.0:1767318615.109601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318616.133310:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318617.160960:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318618.181020:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318619.076763:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318619.076774:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318619.076880:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982606336:0@lo:400:kworker.0 00000100:00100000:1.0:1767318619.076952:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318619.077067:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606336 00000100:00100000:2.0:1767318619.077078:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606336 00000100:00100000:2.0:1767318619.077151:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 623 00000100:00100000:2.0:1767318619.077162:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982606336:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318619.077214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318619.077235:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318619.077240:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318619.077244:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318619.077249:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3862 00000100:00080000:3.0:1767318619.077252:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318619.077257:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318619.077304:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3480 x1853163982606464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318619.077340:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318619.077343:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318619.077345:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318619.077349:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3862 00000100:00080000:3.0:1767318619.077351:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318619.077355:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318619.077365:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3b80 x1853163982606720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318619.077374:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318619.077387:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318619.077389:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318619.077392:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3862 00000100:00080000:3.0:1767318619.077395:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318619.077398:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318619.077409:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2a00 x1853163982606848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318619.077477:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982607104:0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.077509:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077545:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982606592:0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.077569:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077585:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318619.077591:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982606976:0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.077611:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077636:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606592 00000100:00100000:0.0:1767318619.077639:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606592 00000100:00100000:0.0:1767318619.077670:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 624 00000100:00100000:0.0:1767318619.077676:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982606592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.077722:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982606592:12345-0@lo:400:kworker.0 Request processed in 45us (151us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318619.077729:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 624 00000100:00080000:0.0:1767318619.077750:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318619.077773:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1880 x1853163982606592/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318635 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318619.077785:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982606592:0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.077829:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982606464:0@lo:38:kworker.0 00000100:00100000:0.0:1767318619.077852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077867:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982606720:0@lo:38:kworker.0 00000100:00100000:0.0:1767318619.077887:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077901:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982606848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318619.077922:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.077949:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606848 00000100:00100000:0.0:1767318619.077953:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606848 00000100:00100000:0.0:1767318619.077972:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 228 00000100:00100000:0.0:1767318619.077976:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982606848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318619.077993:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318619.078025:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982606848:12345-0@lo:38:kworker.0 Request processed in 48us (102us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318619.078042:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 228 00000100:00100000:0.0:1767318619.078063:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606464 00000100:00100000:0.0:1767318619.078066:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606464 00000100:00100000:0.0:1767318619.078082:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606720 00000100:00100000:0.0:1767318619.078084:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606720 00000100:00100000:0.0:1767318619.078096:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 810 00000100:00100000:0.0:1767318619.078100:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982606464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318619.078109:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318619.078132:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982606464:12345-0@lo:38:kworker.0 Request processed in 31us (279us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318619.078138:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 810 00000100:00100000:0.0:1767318619.078148:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 811 00000100:00100000:0.0:1767318619.078151:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982606720:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318619.078159:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318619.078177:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982606720:12345-0@lo:38:kworker.0 Request processed in 24us (289us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318619.078181:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 811 00000100:00080000:0.0:1767318619.078209:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318619.078213:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318619.078216:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982606464:0@lo:38:kworker.0 00000100:00080000:0.0:1767318619.078224:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318619.078227:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318619.078229:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982606720:0@lo:38:kworker.0 00000100:00080000:0.0:1767318619.078236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318619.078239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318619.078241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982606848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318619.078292:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607104 00000100:00100000:0.0:1767318619.078295:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607104 00000100:00100000:0.0:1767318619.078325:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982606976 00000100:00100000:0.0:1767318619.078328:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982606976 00000100:00100000:0.0:1767318619.078347:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 407 00000100:00100000:0.0:1767318619.078352:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982607104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.078384:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982607104:12345-0@lo:400:kworker.0 Request processed in 32us (874us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318619.078390:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 407 00000100:00100000:0.0:1767318619.078401:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 408 00000100:00100000:0.0:1767318619.078407:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982606976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.078435:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982606976:12345-0@lo:400:kworker.0 Request processed in 28us (823us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318619.078440:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 408 00000100:00080000:0.0:1767318619.078453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318619.078456:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318619.078471:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982607104:0@lo:400:kworker.0 00000100:00100000:0.0:1767318619.078491:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982606976:0@lo:400:kworker.0 00000100:00100000:2.0:1767318619.088323:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982606336:12345-0@lo:400:kworker.0 Request processed in 11140us (11347us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318619.088338:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 623 00010000:00080000:1.0:1767318619.090088:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2300 x1853163982606336/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318635 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318619.090105:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982606336:0@lo:400:kworker.0 00000100:00080000:3.0:1767318619.091642:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318619.091651:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318619.091655:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318619.091662:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3862 00000100:00080000:3.0:1767318619.091667:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318619.091673:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318619.091712:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2300 x1853163982607232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318619.091842:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982607232:0@lo:38:kworker.0 00000100:00100000:0.0:1767318619.091908:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318619.092566:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607232 00000100:00100000:2.0:1767318619.092575:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607232 00000100:00100000:2.0:1767318619.092624:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 812 00000100:00100000:2.0:1767318619.092631:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982607232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318619.092651:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318619.092713:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982607232:12345-0@lo:38:kworker.0 Request processed in 80us (804us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318619.092720:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 812 00000100:00080000:3.0:1767318619.092767:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318619.092772:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318619.092776:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982607232:0@lo:38:kworker.0 00010000:00010000:1.0:1767318619.206587:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318619.588631:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318619.588640:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318619.588657:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982607360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318619.588657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982607488:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318619.610305:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318619.610470:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607360 00000100:00100000:2.0:1767318619.610481:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607360 00000100:00100000:2.0:1767318619.610541:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 438 00000100:00100000:2.0:1767318619.610551:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982607360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318619.610585:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318619.610650:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982607360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (351us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318619.610657:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 438 00000100:00100000:1.0:1767318619.611034:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982607360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318619.615599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318619.615664:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607488 00000100:00100000:0.0:1767318619.615674:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607488 00000100:00100000:0.0:1767318619.615744:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 439 00000100:00100000:0.0:1767318619.615755:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982607488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318619.615790:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318619.615846:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982607488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318619.615854:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 439 00000100:00100000:3.0:1767318619.616245:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982607488:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318620.055857:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318620.055971:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318620.056148:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578880 00000100:00100000:0.0:1767318620.056159:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578880 00000100:00100000:0.0:1767318620.056236:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915578752 00000100:00100000:0.0:1767318620.056240:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915578752 00000100:00100000:0.0:1767318620.056264:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 625 00000100:00100000:0.0:1767318620.056305:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915578880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318620.056447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318620.056476:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 813 00000100:00100000:1.0:1767318620.056482:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915578752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318620.056502:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318620.056562:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915578752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 78us (591us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318620.056569:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 813 00000100:00100000:0.0:1767318620.056870:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915578880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 596us (1027us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318620.056878:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 625 00010000:00010000:1.0:1767318620.233931:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318621.252882:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318622.279713:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318623.300771:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318624.197334:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318624.197346:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318624.197410:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318624.197414:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318624.197418:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318624.197426:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3867 00000100:00080000:3.0:1767318624.197431:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318624.197438:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318624.197459:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1500 x1853163982607744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318624.197498:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318624.197504:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767318624.197506:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982607616:0@lo:400:kworker.0 00000100:00080000:3.0:1767318624.197508:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318624.197525:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3867 00000100:00080000:3.0:1767318624.197528:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318624.197533:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318624.197551:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3b80 x1853163982608000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318624.197561:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318624.197564:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318624.197566:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318624.197569:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3867 00000100:00080000:3.0:1767318624.197572:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318624.197576:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318624.197589:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3480 x1853163982608128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318624.197629:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318624.197630:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318624.197639:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318624.197641:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318624.197645:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3867 00000100:00080000:3.0:1767318624.197649:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318624.197654:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318624.197666:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982608512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318624.197699:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318624.197709:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982608384:0@lo:400:kworker.0 00000100:00100000:3.0:1767318624.197727:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982607872:0@lo:400:kworker.0 00000100:00100000:1.0:1767318624.197751:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.197848:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982607744:0@lo:38:kworker.0 00000100:00100000:2.0:1767318624.197981:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982608256:0@lo:400:kworker.0 00000100:00100000:1.0:1767318624.201316:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607616 00000100:00100000:1.0:1767318624.201338:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607616 00000100:00100000:3.0:1767318624.202699:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318624.202820:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607872 00000100:00100000:3.0:1767318624.202831:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607872 00000100:00100000:0.0:1767318624.204563:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.204621:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982608000:0@lo:38:kworker.0 00000100:00100000:0.0:1767318624.204666:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.204816:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982608128:0@lo:38:kworker.0 00000100:00100000:0.0:1767318624.204852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.204903:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982608512:0@lo:38:kworker.0 00000100:00100000:0.0:1767318624.204942:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318624.204945:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982607744 00000100:00100000:1.0:1767318624.204961:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982607744 00000100:00100000:1.0:1767318624.204984:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608000 00000100:00100000:1.0:1767318624.204988:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608000 00000100:00100000:1.0:1767318624.205003:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608512 00000100:00100000:1.0:1767318624.205007:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608512 00000100:00100000:1.0:1767318624.205037:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 626 00000100:00100000:1.0:1767318624.205243:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982607616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318624.205337:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 814 00000100:00100000:0.0:1767318624.205348:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982607744:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318624.205450:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 627 00000100:00100000:3.0:1767318624.205462:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982607872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318624.205529:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982607872:12345-0@lo:400:kworker.0 Request processed in 71us (2840us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318624.205535:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 627 00000100:00100000:3.0:1767318624.205553:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 815 00000100:00100000:1.0:1767318624.205560:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982607616:12345-0@lo:400:kworker.0 Request processed in 322us (7929us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318624.205568:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982608000:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318624.205579:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 626 00010000:00020000:0.0:1767318624.205603:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767318624.205659:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318624.205700:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982607744:12345-0@lo:38:kworker.0 Request processed in 353us (1143us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318624.205727:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 816 00000100:00100000:1.0:1767318624.205737:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982608512:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318624.205748:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982608000:12345-0@lo:38:kworker.0 Request processed in 182us (1078us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767318624.205758:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318624.205772:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 814 00000100:00100000:2.0:1767318624.205783:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.205821:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608128 00000100:00100000:0.0:1767318624.205830:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608128 00000100:00100000:0.0:1767318624.205863:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 229 00000100:00100000:0.0:1767318624.205868:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982608128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318624.205880:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318624.205888:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608384 00000100:00100000:2.0:1767318624.205896:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608384 00000100:00100000:0.0:1767318624.205917:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982608128:12345-0@lo:38:kworker.0 Request processed in 46us (1062us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318624.205926:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 229 00000100:00100000:3.0:1767318624.205966:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 815 00000100:00100000:2.0:1767318624.206019:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608256 00010000:00080000:3.0:1767318624.206020:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2a00 x1853163982607872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318640 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318624.206027:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608256 00000100:00100000:3.0:1767318624.206045:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982607872:0@lo:400:kworker.0 00000100:00080000:3.0:1767318624.206107:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318624.206112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318624.206115:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318624.213100:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982608512:12345-0@lo:38:kworker.0 Request processed in 7347us (8153us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318624.213121:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 816 00000100:00080000:1.0:1767318624.213322:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318624.213329:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318624.213333:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982607744:0@lo:38:kworker.0 00000100:00080000:1.0:1767318624.213349:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318624.213352:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318624.213355:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982608000:0@lo:38:kworker.0 00000100:00080000:1.0:1767318624.213368:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318624.213371:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318624.213374:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982608128:0@lo:38:kworker.0 00000100:00080000:1.0:1767318624.213384:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318624.213387:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318624.213390:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982608512:0@lo:38:kworker.0 00000100:00100000:0.0:1767318624.213558:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 409 00000100:00100000:0.0:1767318624.213578:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 410 00000100:00100000:0.0:1767318624.213594:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982608256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318624.213702:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982608256:12345-0@lo:400:kworker.0 Request processed in 117us (7928us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318624.213709:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 410 00000100:00100000:0.0:1767318624.213733:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982608384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318624.213764:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982608384:12345-0@lo:400:kworker.0 Request processed in 198us (16016us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318624.213771:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 409 00000100:00100000:1.0:1767318624.220389:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982608256:0@lo:400:kworker.0 00010000:00080000:3.0:1767318624.220667:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2300 x1853163982607616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318640 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318624.220688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982607616:0@lo:400:kworker.0 00000100:00100000:3.0:1767318624.220723:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982608384:0@lo:400:kworker.0 00010000:00010000:3.0:1767318624.326977:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318624.708752:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982608768:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318624.708777:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318624.708805:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982608640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318624.708855:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318624.708895:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318624.709217:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608768 00000100:00100000:0.0:1767318624.709229:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608768 00000100:00100000:0.0:1767318624.709337:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608640 00000100:00100000:0.0:1767318624.709341:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608640 00000100:00100000:0.0:1767318624.709363:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 440 00000100:00100000:0.0:1767318624.709374:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982608768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318624.709415:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318624.709484:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982608768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (631us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318624.709496:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 440 00000100:00100000:0.0:1767318624.709518:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 441 00000100:00100000:0.0:1767318624.709525:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982608640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318624.709544:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318624.709567:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982608768:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318624.709581:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982608640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (685us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318624.709593:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 441 00000100:00100000:2.0:1767318624.716001:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982608640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318625.166794:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318625.170179:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318625.170372:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579136 00000100:00100000:3.0:1767318625.170388:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579136 00000100:00100000:3.0:1767318625.170506:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579008 00000100:00100000:3.0:1767318625.170510:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579008 00000100:00100000:3.0:1767318625.170540:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 628 00000100:00100000:3.0:1767318625.170552:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915579136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318625.170653:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915579136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (3877us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318625.170661:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 628 00000100:00100000:3.0:1767318625.170683:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 817 00000100:00100000:3.0:1767318625.170688:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915579008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318625.170723:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318625.170752:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915579008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 62us (588us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318625.170759:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 817 00000100:00080000:1.0:1767318625.171001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318625.348804:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318626.374790:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318627.396599:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318628.421673:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318629.320029:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318629.320043:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318629.320110:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318629.320115:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318629.320120:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318629.320132:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3872 00000100:00080000:1.0:1767318629.320138:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318629.320147:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318629.320174:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583800 x1853163982609024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318629.320223:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318629.320227:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318629.320229:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318629.320234:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3872 00000100:00080000:1.0:1767318629.320237:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318629.320242:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318629.320253:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582680 x1853163982609280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318629.320294:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318629.320298:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318629.320300:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318629.320305:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3872 00000100:00080000:1.0:1767318629.320307:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318629.320312:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318629.320329:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163982609408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318629.320396:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318629.320401:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318629.320403:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318629.320408:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3872 00000100:00080000:1.0:1767318629.320411:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318629.320415:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318629.320429:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1c00 x1853163982609792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318629.320473:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982609664:0@lo:400:kworker.0 00000100:00100000:1.0:1767318629.320490:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982609536:0@lo:400:kworker.0 00000100:00100000:0.0:1767318629.320717:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982609024:0@lo:38:kworker.0 00000100:00100000:1.0:1767318629.320784:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318629.320868:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318629.320874:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982609152:0@lo:400:kworker.0 00000100:00100000:1.0:1767318629.320903:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318629.322840:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982608896:0@lo:400:kworker.0 00000100:00100000:2.0:1767318629.322942:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318629.323063:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609536 00000100:00100000:2.0:1767318629.323075:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609536 00000100:00100000:2.0:1767318629.323147:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 411 00000100:00100000:2.0:1767318629.323158:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982609536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318629.323234:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982609536:12345-0@lo:400:kworker.0 Request processed in 77us (2450us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318629.323243:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 411 00000100:00100000:2.0:1767318629.323317:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609152 00000100:00100000:2.0:1767318629.323322:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609152 00000100:00100000:2.0:1767318629.323367:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982608896 00000100:00100000:2.0:1767318629.323371:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982608896 00000100:00100000:2.0:1767318629.323396:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 629 00000100:00100000:2.0:1767318629.323402:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982609152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318629.323440:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982609152:12345-0@lo:400:kworker.0 Request processed in 38us (2535us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318629.323446:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 629 00000100:00100000:2.0:1767318629.323457:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 630 00000100:00100000:2.0:1767318629.323477:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982608896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318629.323519:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982608896:12345-0@lo:400:kworker.0 Request processed in 43us (576us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318629.323525:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 630 00010000:00080000:2.0:1767318629.323571:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580000 x1853163982608896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318645 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318629.324056:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982608896:0@lo:400:kworker.0 00000100:00080000:1.0:1767318629.325173:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318629.325187:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:1.0:1767318629.325264:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581c00 x1853163982609152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318645 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318629.325332:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982609152:0@lo:400:kworker.0 00000100:00100000:1.0:1767318629.325359:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982609536:0@lo:400:kworker.0 00000100:00100000:3.0:1767318629.330312:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318629.330330:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318629.330968:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982609280:0@lo:38:kworker.0 00000100:00100000:0.0:1767318629.331061:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318629.331123:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609664 00000100:00100000:1.0:1767318629.331132:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609664 00000100:00100000:1.0:1767318629.331215:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 412 00000100:00100000:1.0:1767318629.331225:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982609664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318629.331303:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982609664:12345-0@lo:400:kworker.0 Request processed in 79us (1009us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318629.331310:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 412 00000100:00100000:0.0:1767318629.331565:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982609408:0@lo:38:kworker.0 00000100:00100000:0.0:1767318629.331607:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318629.331630:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982609792:0@lo:38:kworker.0 00000100:00100000:0.0:1767318629.331654:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318629.331702:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609408 00000100:00100000:1.0:1767318629.331707:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609408 00000100:00100000:1.0:1767318629.331732:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 230 00000100:00100000:1.0:1767318629.331737:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982609408:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318629.331757:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318629.331788:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982609408:12345-0@lo:38:kworker.0 Request processed in 50us (181us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318629.331795:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 230 00000100:00080000:3.0:1767318629.331867:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318629.331906:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982609664:0@lo:400:kworker.0 00000100:00100000:2.0:1767318629.332883:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609024 00000100:00100000:2.0:1767318629.332897:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609024 00000100:00100000:2.0:1767318629.332966:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609280 00000100:00100000:2.0:1767318629.332971:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609280 00000100:00100000:2.0:1767318629.333142:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 818 00000100:00100000:2.0:1767318629.333150:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982609024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318629.333181:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318629.333237:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982609024:12345-0@lo:38:kworker.0 Request processed in 87us (2906us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318629.333431:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 818 00000100:00100000:2.0:1767318629.333451:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 819 00000100:00100000:2.0:1767318629.333457:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982609280:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318629.333566:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318629.333823:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982609280:12345-0@lo:38:kworker.0 Request processed in 364us (2762us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318629.333839:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 819 00000100:00080000:0.0:1767318629.334448:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318629.334456:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318629.334460:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982609024:0@lo:38:kworker.0 00000100:00100000:3.0:1767318629.334659:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609792 00000100:00100000:3.0:1767318629.334667:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609792 00000100:00100000:3.0:1767318629.334697:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 820 00000100:00100000:3.0:1767318629.334703:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982609792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318629.334721:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318629.334759:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982609792:12345-0@lo:38:kworker.0 Request processed in 56us (3104us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318629.334765:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 820 00000100:00080000:0.0:1767318629.335169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318629.335175:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318629.335179:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982609280:0@lo:38:kworker.0 00000100:00080000:0.0:1767318629.335196:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318629.335201:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318629.335204:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982609408:0@lo:38:kworker.0 00000100:00080000:0.0:1767318629.335214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318629.335217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318629.335220:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982609792:0@lo:38:kworker.0 00010000:00010000:1.0:1767318629.444645:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318629.828587:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318629.828614:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982609920:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318629.828694:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318629.828854:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982609920 00000100:00100000:1.0:1767318629.828865:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982609920 00000100:00100000:1.0:1767318629.828929:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 442 00000100:00100000:1.0:1767318629.828938:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982609920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318629.828969:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318629.829023:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982609920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318629.829031:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 442 00000100:00100000:3.0:1767318629.829456:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982609920:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318629.831043:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318629.831063:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982610048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318629.831148:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318629.831654:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610048 00000100:00100000:0.0:1767318629.831664:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610048 00000100:00100000:0.0:1767318629.831730:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 443 00000100:00100000:0.0:1767318629.831741:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982610048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318629.831778:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318629.831848:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982610048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (711us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318629.831856:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 443 00000100:00100000:1.0:1767318629.832007:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982610048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318630.289019:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318630.289119:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579392 00000100:00100000:2.0:1767318630.289131:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579392 00000100:00100000:2.0:1767318630.289309:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 631 00000100:00100000:2.0:1767318630.289321:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915579392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318630.289394:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915579392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (386us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318630.289401:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 631 00000100:00080000:3.0:1767318630.292098:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318630.292179:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318630.292584:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579264 00000100:00100000:2.0:1767318630.292647:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579264 00000100:00100000:2.0:1767318630.292687:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 821 00000100:00100000:2.0:1767318630.292694:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915579264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318630.292713:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318630.292874:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915579264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 179us (701us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318630.292881:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 821 00010000:00010000:3.0:1767318630.468551:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318631.494445:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318632.517306:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318633.543332:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318634.436379:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318634.436391:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318634.436464:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318634.436469:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318634.436473:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318634.436481:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3877 00000100:00080000:1.0:1767318634.436487:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318634.436494:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318634.436518:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1880 x1853163982610304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318634.436518:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982610176:0@lo:400:kworker.0 00000100:00100000:3.0:1767318634.436592:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318634.441085:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318634.441096:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318634.441102:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318634.441112:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3877 00000100:00080000:1.0:1767318634.441117:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318634.441125:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318634.441169:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3480 x1853163982610560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318634.441191:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318634.441194:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318634.441196:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318634.441200:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3877 00000100:00080000:1.0:1767318634.441202:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318634.441205:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318634.441217:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1500 x1853163982610688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318634.441265:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318634.441285:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318634.441288:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318634.441291:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3877 00000100:00080000:1.0:1767318634.441294:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318634.441297:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318634.441313:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3100 x1853163982611072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318634.441381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982610304:0@lo:38:kworker.0 00000100:00100000:1.0:1767318634.441651:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318634.442022:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982610560:0@lo:38:kworker.0 00000100:00100000:1.0:1767318634.442053:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318634.442073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982610688:0@lo:38:kworker.0 00000100:00100000:1.0:1767318634.442097:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318634.442120:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982611072:0@lo:38:kworker.0 00000100:00100000:1.0:1767318634.442141:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318634.442187:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610176 00000100:00100000:1.0:1767318634.442197:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610176 00000100:00100000:1.0:1767318634.442251:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610304 00000100:00100000:1.0:1767318634.442254:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610304 00000100:00100000:1.0:1767318634.442266:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610560 00000100:00100000:1.0:1767318634.442268:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610560 00000100:00100000:1.0:1767318634.442309:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611072 00000100:00100000:1.0:1767318634.442312:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611072 00000100:00100000:1.0:1767318634.442325:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 632 00000100:00100000:1.0:1767318634.442334:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982610176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318634.442382:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982610176:12345-0@lo:400:kworker.0 Request processed in 50us (5793us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318634.442390:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 632 00000100:00100000:1.0:1767318634.442409:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 822 00000100:00100000:1.0:1767318634.442413:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982610304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318634.442430:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318634.442581:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982610944:0@lo:400:kworker.0 00000100:00100000:2.0:1767318634.442617:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318634.442660:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 823 00000100:00100000:2.0:1767318634.442667:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982610560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318634.442679:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318634.442688:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982610304:12345-0@lo:38:kworker.0 Request processed in 263us (1028us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318634.442697:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 822 00000100:00100000:2.0:1767318634.442712:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982610560:12345-0@lo:38:kworker.0 Request processed in 43us (657us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318634.442717:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 824 00000100:00100000:2.0:1767318634.442728:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 823 00000100:00100000:1.0:1767318634.442729:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982611072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318634.442755:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318634.442775:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982610432:0@lo:400:kworker.0 00000100:00100000:1.0:1767318634.442787:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982611072:12345-0@lo:38:kworker.0 Request processed in 61us (644us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318634.442794:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 824 00000100:00100000:2.0:1767318634.442807:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318634.442818:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610688 00000100:00100000:1.0:1767318634.442824:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610688 00000100:00100000:2.0:1767318634.442830:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318634.442837:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982610816:0@lo:400:kworker.0 00000100:00100000:2.0:1767318634.442857:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767318634.442906:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2d80 x1853163982610176/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318650 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318634.442921:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982610176:0@lo:400:kworker.0 00000100:00100000:1.0:1767318634.442929:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 231 00000100:00100000:1.0:1767318634.442935:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982610688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318634.442946:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318634.442963:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610944 00000100:00100000:2.0:1767318634.442968:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610944 00000100:00100000:1.0:1767318634.442977:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982610688:12345-0@lo:38:kworker.0 Request processed in 40us (878us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318634.442984:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 231 00000100:00100000:2.0:1767318634.443007:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610816 00000100:00100000:1.0:1767318634.443011:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 413 00000100:00100000:2.0:1767318634.443013:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610816 00000100:00100000:1.0:1767318634.443021:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982610944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318634.443042:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 414 00000100:00100000:2.0:1767318634.443051:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982610816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318634.443060:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982610944:12345-0@lo:400:kworker.0 Request processed in 40us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318634.443067:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 413 00000100:00100000:2.0:1767318634.443094:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982610816:12345-0@lo:400:kworker.0 Request processed in 43us (235us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318634.443102:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 414 00000100:00100000:1.0:1767318634.443107:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982610944:0@lo:400:kworker.0 00000100:00080000:2.0:1767318634.443122:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318634.443130:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982610432 00000100:00100000:1.0:1767318634.443134:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982610432 00000100:00100000:1.0:1767318634.443159:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 633 00000100:00100000:1.0:1767318634.443165:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982610432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318634.443194:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982610432:12345-0@lo:400:kworker.0 Request processed in 29us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318634.443200:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 633 00000100:00080000:3.0:1767318634.443252:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318634.443258:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318634.443262:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982610304:0@lo:38:kworker.0 00000100:00080000:3.0:1767318634.443287:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318634.443292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318634.443295:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982610560:0@lo:38:kworker.0 00000100:00080000:3.0:1767318634.443303:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318634.443306:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318634.443308:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982610688:0@lo:38:kworker.0 00000100:00080000:3.0:1767318634.443315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318634.443318:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318634.443320:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982611072:0@lo:38:kworker.0 00000100:00100000:3.0:1767318634.443358:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982610816:0@lo:400:kworker.0 00010000:00080000:3.0:1767318634.443369:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2300 x1853163982610432/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318650 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318634.443376:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982610432:0@lo:400:kworker.0 00000100:00080000:2.0:1767318634.445209:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318634.445222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318634.564522:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318634.948626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982611200:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318634.948651:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318634.948683:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982611328:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318634.948703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318634.948746:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611200 00000100:00100000:0.0:1767318634.948755:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611200 00000100:00100000:0.0:1767318634.948807:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 444 00000100:00100000:0.0:1767318634.948819:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982611200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318634.949010:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318634.949081:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982611200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 256us (370us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318634.949089:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 444 00000100:00100000:0.0:1767318634.949124:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318634.949152:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611328 00000100:00100000:0.0:1767318634.949156:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611328 00000100:00100000:0.0:1767318634.949184:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 445 00000100:00100000:0.0:1767318634.949191:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982611328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318634.949208:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318634.949244:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982611328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (118us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318634.949250:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 445 00000100:00100000:1.0:1767318634.949546:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982611328:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318634.949858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982611200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318635.412426:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318635.412472:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318635.412499:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915579776 offset 0 length 520: 4 00000100:00100000:0.0:1767318635.412707:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579648 00000100:00100000:0.0:1767318635.412728:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579648 00000100:00100000:0.0:1767318635.412812:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579520 00000100:00100000:0.0:1767318635.412816:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579520 00000100:00100000:0.0:1767318635.412839:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 634 00000100:00100000:0.0:1767318635.412856:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915579648:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318635.412941:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915579648:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (529us total) trans 0 rc 0/0 00000100:00080000:2.0:1767318635.413429:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318635.420557:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 825 00000100:00100000:3.0:1767318635.420573:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915579520:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318635.420597:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318635.420916:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915579520:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 344us (8442us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318635.420929:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 825 00000100:00100000:0.0:1767318635.424907:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 634 00010000:00010000:1.0:1767318635.590774:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318636.613129:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318637.639102:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318638.532417:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318613/real 1767318613] req@ffff934604583b80 x1853163982605952/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318638 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318638.532461:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604583b80 x1853163982605952/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318638 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318638.532477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318638.532485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318638.532488:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982605952:0@lo:250:kworker.0 00000100:00080000:2.0:1767318638.532717:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318638.532721:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318638.532724:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318638.532728:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3862 00000100:00080000:2.0:1767318638.532731:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318638.532738:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318638.532783:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163982611456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318638.532834:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982611456:0@lo:250:kworker.0 00000400:00000100:3.0:1767318638.532894:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982611456 offset 0 length 520: 4 00000100:00080000:2.0:1767318638.532979:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318638.532983:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318638.532984:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318638.532988:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3882 00000100:00080000:2.0:1767318638.532990:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318638.532994:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318638.533018:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625880 x1853163982611712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318638.533073:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982611584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318638.533115:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318638.534058:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611584 00000100:00100000:1.0:1767318638.534067:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611584 00000100:00100000:1.0:1767318638.534137:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 635 00000100:00100000:1.0:1767318638.534147:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982611584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318638.534216:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982611584:12345-0@lo:400:kworker.0 Request processed in 72us (1101us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318638.534225:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 635 00010000:00080000:1.0:1767318638.534268:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626d80 x1853163982611584/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318654 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318638.534318:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982611584:0@lo:400:kworker.0 00000100:00080000:0.0:1767318638.537234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318638.540391:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982611712:0@lo:38:kworker.0 00000100:00100000:3.0:1767318638.540486:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318638.540696:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611712 00000100:00100000:0.0:1767318638.540704:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611712 00000100:00100000:0.0:1767318638.540756:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 826 00000100:00100000:0.0:1767318638.540762:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982611712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318638.540786:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318638.540839:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982611712:12345-0@lo:38:kworker.0 Request processed in 76us (352us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318638.540857:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 826 00000100:00080000:3.0:1767318638.541539:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318638.541546:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318638.541550:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982611712:0@lo:38:kworker.0 00000100:00100000:1.0:1767318638.542697:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982611840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318638.542788:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318638.544315:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611840 00000100:00100000:0.0:1767318638.544328:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611840 00000100:00100000:0.0:1767318638.544416:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 636 00000100:00100000:0.0:1767318638.544437:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982611840:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318638.544546:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318638.544782:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318638.544788:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318638.544791:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318638.544795:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3882 00000100:00080000:2.0:1767318638.544799:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318638.544808:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318638.544835:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627480 x1853163982611968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318638.544863:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318638.544867:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318638.544869:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318638.544872:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3882 00000100:00080000:2.0:1767318638.544875:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318638.544878:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318638.544892:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625500 x1853163982612096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318638.544946:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982612224:0@lo:400:kworker.0 00000100:00100000:3.0:1767318638.544986:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318638.545032:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612224 00000100:00100000:3.0:1767318638.545036:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612224 00000100:00100000:3.0:1767318638.545063:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 415 00000100:00100000:3.0:1767318638.545067:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982612224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318638.545092:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982612224:12345-0@lo:400:kworker.0 Request processed in 25us (105us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318638.545097:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 415 00000100:00080000:3.0:1767318638.545112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318638.545131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982612224:0@lo:400:kworker.0 00000100:00100000:3.0:1767318638.545146:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982611968:0@lo:38:kworker.0 00000100:00100000:3.0:1767318638.545162:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318638.545173:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982612096:0@lo:38:kworker.0 00000100:00100000:3.0:1767318638.545186:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318638.545203:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612096 00000100:00100000:3.0:1767318638.545206:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612096 00000100:00100000:3.0:1767318638.545219:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 232 00000100:00100000:3.0:1767318638.545222:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982612096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318638.545237:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318638.545263:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982612096:12345-0@lo:38:kworker.0 Request processed in 39us (75us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318638.545287:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 232 00000100:00100000:3.0:1767318638.545304:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982611968 00000100:00100000:3.0:1767318638.545308:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982611968 00000100:00100000:3.0:1767318638.545329:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827 00000100:00100000:3.0:1767318638.545333:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982611968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318638.545341:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318638.545365:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982611968:12345-0@lo:38:kworker.0 Request processed in 31us (201us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318638.545371:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827 00000100:00080000:3.0:1767318638.545397:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318638.545402:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318638.545405:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982611968:0@lo:38:kworker.0 00000100:00080000:3.0:1767318638.545414:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318638.545417:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318638.545420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982612096:0@lo:38:kworker.0 00000100:00100000:0.0:1767318638.548258:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982611840:12345-0@lo:400:kworker.0 Request processed in 3821us (5466us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318638.548285:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 636 00010000:00080000:1.0:1767318638.548380:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626300 x1853163982611840/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318654 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318638.548396:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982611840:0@lo:400:kworker.0 00000100:00100000:1.0:1767318638.548688:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982612352:0@lo:400:kworker.0 00000100:00100000:1.0:1767318638.548732:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318638.549100:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318638.549104:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318638.549106:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318638.549110:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3882 00000100:00080000:2.0:1767318638.549113:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318638.549118:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318638.549135:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626300 x1853163982612480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318638.549183:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982612480:0@lo:38:kworker.0 00000100:00100000:3.0:1767318638.549207:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318638.549255:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612480 00000100:00100000:0.0:1767318638.549261:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612480 00000100:00100000:0.0:1767318638.549306:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 828 00000100:00100000:0.0:1767318638.549313:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982612480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318638.549331:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318638.549415:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612352 00000100:00100000:1.0:1767318638.549419:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612352 00000100:00100000:1.0:1767318638.549475:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 416 00000100:00100000:1.0:1767318638.549480:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982612352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318638.549511:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982612352:12345-0@lo:400:kworker.0 Request processed in 32us (778us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318638.549515:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 416 00000100:00100000:1.0:1767318638.549535:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982612352:0@lo:400:kworker.0 00000100:00080000:1.0:1767318638.549545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318638.556253:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982612480:12345-0@lo:38:kworker.0 Request processed in 6446us (6550us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318638.556290:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 828 00000100:00080000:3.0:1767318638.556571:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318638.556578:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318638.556582:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982612480:0@lo:38:kworker.0 00010000:00010000:1.0:1767318638.662000:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318639.684471:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318640.068872:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982612608:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318640.068958:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318640.069197:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318640.069216:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982612736:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318640.069261:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318640.073189:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612608 00000100:00100000:0.0:1767318640.073204:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612608 00000100:00100000:0.0:1767318640.073302:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612736 00000100:00100000:0.0:1767318640.073307:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612736 00000100:00100000:0.0:1767318640.073344:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 446 00000100:00100000:0.0:1767318640.073355:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982612608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318640.073416:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318640.073539:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982612608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 184us (4580us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318640.073548:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 446 00000100:00100000:0.0:1767318640.073577:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 447 00000100:00100000:0.0:1767318640.073584:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982612736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318640.073616:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318640.073658:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982612736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (4409us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318640.073663:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 447 00000100:00080000:3.0:1767318640.073694:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318640.073702:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318640.073734:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982612608:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318640.073738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982612736:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318640.523761:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318640.523859:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915579904 00000100:00100000:0.0:1767318640.523869:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915579904 00000100:00100000:0.0:1767318640.523918:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 829 00000100:00100000:0.0:1767318640.523933:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915579904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318640.523959:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318640.524028:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915579904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (276us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318640.524035:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 829 00000100:00100000:2.0:1767318640.524690:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318640.524750:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580032 00000100:00100000:3.0:1767318640.524755:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580032 00000100:00100000:3.0:1767318640.524815:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 637 00000100:00100000:3.0:1767318640.524823:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915580032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318640.524873:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915580032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 52us (185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318640.524880:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 637 00000100:00080000:1.0:1767318640.525088:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318640.709087:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318641.733626:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318642.757116:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318643.654091:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318643.654103:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318643.654190:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318643.654195:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318643.654199:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318643.654208:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3887 00000100:00080000:3.0:1767318643.654214:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318643.654222:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318643.654264:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982612864:0@lo:400:kworker.0 00000100:00100000:3.0:1767318643.654308:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2300 x1853163982612992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318643.654344:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318643.654347:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318643.654349:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318643.654352:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3887 00000100:00080000:3.0:1767318643.654354:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318643.654358:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318643.654369:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1500 x1853163982613248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318643.654376:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318643.654377:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318643.654379:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318643.654381:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318643.654384:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3887 00000100:00080000:3.0:1767318643.654387:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318643.654390:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318643.654400:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0700 x1853163982613376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318643.654419:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318643.654422:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318643.654424:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318643.654427:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3887 00000100:00080000:3.0:1767318643.654429:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318643.654432:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318643.654435:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612864 00000100:00100000:0.0:1767318643.654441:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612864 00000100:00100000:3.0:1767318643.654441:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163982613760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318643.654463:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982613120:0@lo:400:kworker.0 00000100:00100000:3.0:1767318643.654490:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318643.654506:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613120 00000100:00100000:0.0:1767318643.654510:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613120 00000100:00100000:3.0:1767318643.654518:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318643.654524:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982613504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654533:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 638 00000100:00100000:0.0:1767318643.654540:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982612864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318643.654547:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318643.654588:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 639 00000100:00100000:3.0:1767318643.654595:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982613120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654596:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982612864:12345-0@lo:400:kworker.0 Request processed in 56us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318643.654603:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 638 00000100:00080000:0.0:1767318643.654623:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318643.654624:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982613120:12345-0@lo:400:kworker.0 Request processed in 30us (134us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318643.654629:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 639 00000100:00100000:3.0:1767318643.654651:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613504 00010000:00080000:0.0:1767318643.654654:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1500 x1853163982612864/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318659 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318643.654654:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613504 00000100:00100000:0.0:1767318643.654666:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982612864:0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654678:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318643.654683:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 417 00000100:00100000:0.0:1767318643.654684:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982613632:0@lo:400:kworker.0 00000100:00100000:3.0:1767318643.654689:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982613504:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654704:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318643.654714:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982613504:12345-0@lo:400:kworker.0 Request processed in 26us (167us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318643.654723:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 417 00000100:00100000:0.0:1767318643.654726:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613632 00000100:00100000:0.0:1767318643.654729:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613632 00000100:00100000:0.0:1767318643.654750:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 418 00000100:00100000:3.0:1767318643.654750:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982613504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654755:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982613632:12345-0@lo:400:kworker.0 00010000:00080000:3.0:1767318643.654758:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3480 x1853163982613120/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318659 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318643.654767:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982613120:0@lo:400:kworker.0 00000100:00100000:0.0:1767318643.654785:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982613632:12345-0@lo:400:kworker.0 Request processed in 30us (80us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318643.654790:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 418 00000100:00080000:0.0:1767318643.654802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318643.654804:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318643.654818:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982613632:0@lo:400:kworker.0 00000100:00100000:1.0:1767318643.655025:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982612992:0@lo:38:kworker.0 00000100:00100000:1.0:1767318643.655054:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318643.655086:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982613248:0@lo:38:kworker.0 00000100:00100000:1.0:1767318643.655105:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318643.655127:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982613376:0@lo:38:kworker.0 00000100:00100000:1.0:1767318643.655151:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318643.655163:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982613760:0@lo:38:kworker.0 00000100:00100000:1.0:1767318643.655180:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318643.655296:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982612992 00000100:00100000:0.0:1767318643.655301:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982612992 00000100:00100000:0.0:1767318643.655326:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613248 00000100:00100000:0.0:1767318643.655330:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613248 00000100:00100000:0.0:1767318643.655349:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613760 00000100:00100000:0.0:1767318643.655352:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613760 00000100:00100000:0.0:1767318643.655362:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830 00000100:00100000:0.0:1767318643.655367:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982612992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318643.655390:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318643.655427:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982612992:12345-0@lo:38:kworker.0 Request processed in 60us (372us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318643.655435:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830 00000100:00100000:0.0:1767318643.655446:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767318643.655449:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982613248:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318643.655457:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318643.655477:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982613248:12345-0@lo:38:kworker.0 Request processed in 27us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318643.655482:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831 00000100:00100000:2.0:1767318643.655590:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 832 00000100:00100000:2.0:1767318643.655596:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982613760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318643.655605:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318643.655636:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982613760:12345-0@lo:38:kworker.0 Request processed in 38us (454us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318643.655641:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 832 00000100:00080000:2.0:1767318643.655684:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318643.655689:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318643.655692:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982612992:0@lo:38:kworker.0 00000100:00080000:2.0:1767318643.655700:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318643.655703:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318643.655705:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982613248:0@lo:38:kworker.0 00000100:00080000:2.0:1767318643.655712:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318643.655715:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318643.655718:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982613760:0@lo:38:kworker.0 00000100:00100000:3.0:1767318643.655864:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613376 00000100:00100000:3.0:1767318643.655869:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613376 00000100:00100000:3.0:1767318643.655907:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 233 00000100:00100000:3.0:1767318643.655912:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982613376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318643.655921:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318643.655958:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982613376:12345-0@lo:38:kworker.0 Request processed in 45us (806us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318643.655964:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 233 00000100:00080000:2.0:1767318643.656115:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318643.656120:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318643.656123:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982613376:0@lo:38:kworker.0 00010000:00010000:1.0:1767318643.780418:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318644.804963:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318645.188714:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982613888:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318645.188849:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318645.189293:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982613888 00000100:00100000:0.0:1767318645.189302:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982613888 00000100:00100000:0.0:1767318645.189376:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 448 00000100:00100000:0.0:1767318645.189386:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982613888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318645.189420:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318645.189489:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982613888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (640us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318645.189497:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 448 00000100:00100000:0.0:1767318645.189557:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982613888:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318645.191422:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318645.193370:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318645.193394:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982614016:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318645.193481:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318645.194541:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614016 00000100:00100000:0.0:1767318645.194552:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614016 00000100:00100000:0.0:1767318645.194636:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 449 00000100:00100000:0.0:1767318645.194648:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982614016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318645.194687:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318645.194776:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982614016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 130us (1295us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318645.194783:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 449 00000100:00100000:2.0:1767318645.195083:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982614016:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318645.195302:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318645.645638:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318645.645688:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318645.645790:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580288 00000100:00100000:3.0:1767318645.645801:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580288 00000100:00100000:3.0:1767318645.645872:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580160 00000100:00100000:3.0:1767318645.645875:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580160 00000100:00100000:3.0:1767318645.645898:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 640 00000100:00100000:3.0:1767318645.645910:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915580288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318645.645967:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 833 00000100:00100000:0.0:1767318645.645975:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915580160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318645.646000:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318645.646024:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915580288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (398us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318645.646035:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 640 00000100:00100000:0.0:1767318645.646042:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915580160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (351us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318645.646051:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 833 00000100:00080000:1.0:1767318645.646873:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318645.828482:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318646.852475:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318647.876580:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318648.774125:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318648.774135:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318648.774252:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982614144:0@lo:400:kworker.0 00000100:00100000:2.0:1767318648.774363:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318648.774496:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614144 00000100:00100000:3.0:1767318648.774504:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614144 00000100:00100000:3.0:1767318648.774572:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 641 00000100:00100000:3.0:1767318648.774583:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982614144:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318648.774677:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318648.774875:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318648.774880:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318648.774884:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318648.774889:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318648.774893:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318648.774898:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318648.774921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964380 x1853163982614272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318648.774951:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318648.774954:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318648.774956:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318648.774959:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318648.774962:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318648.774965:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318648.774975:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967800 x1853163982614528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318648.774982:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318648.774985:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318648.774987:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318648.774990:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3892 00000100:00100000:0.0:1767318648.775106:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318648.775118:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982614400:0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.775149:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.775175:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614400 00000100:00100000:0.0:1767318648.775180:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614400 00000100:00100000:0.0:1767318648.775209:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 642 00000100:00100000:0.0:1767318648.775215:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982614400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.775247:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982614400:12345-0@lo:400:kworker.0 Request processed in 32us (98us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318648.775254:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 642 00000100:00080000:0.0:1767318648.775291:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318648.775331:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964000 x1853163982614400/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318664 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318648.775344:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982614400:0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.775382:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982614272:0@lo:38:kworker.0 00000100:00100000:0.0:1767318648.775405:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.775421:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982614528:0@lo:38:kworker.0 00000100:00100000:0.0:1767318648.775437:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.775472:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614272 00000100:00100000:0.0:1767318648.775477:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614272 00000100:00100000:0.0:1767318648.775494:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614528 00000100:00100000:0.0:1767318648.775496:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614528 00000100:00100000:0.0:1767318648.775505:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 834 00000100:00100000:0.0:1767318648.775509:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982614272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318648.775528:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318648.775569:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982614272:12345-0@lo:38:kworker.0 Request processed in 58us (163us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318648.775576:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 834 00000100:00100000:0.0:1767318648.775587:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 835 00000100:00100000:0.0:1767318648.775591:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982614528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318648.775600:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318648.775621:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982614528:12345-0@lo:38:kworker.0 Request processed in 29us (183us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318648.775626:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 835 00000100:00080000:0.0:1767318648.775651:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318648.775656:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318648.775659:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982614272:0@lo:38:kworker.0 00000100:00080000:0.0:1767318648.775667:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318648.775669:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318648.775672:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982614528:0@lo:38:kworker.0 00000100:00080000:1.0:1767318648.775758:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318648.775763:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318648.775780:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965c00 x1853163982614656/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318648.775895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982614656:0@lo:38:kworker.0 00000100:00100000:0.0:1767318648.775923:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.775959:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614656 00000100:00100000:0.0:1767318648.775963:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614656 00000100:00100000:0.0:1767318648.775982:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 234 00000100:00100000:0.0:1767318648.775987:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982614656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318648.776010:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318648.776038:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982614656:12345-0@lo:38:kworker.0 Request processed in 50us (114us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318648.776045:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 234 00000100:00080000:0.0:1767318648.776064:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318648.776068:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318648.776071:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982614656:0@lo:38:kworker.0 00000100:00100000:0.0:1767318648.776091:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318648.776099:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982614784:0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.776125:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.776147:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614784 00000100:00100000:0.0:1767318648.776151:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614784 00000100:00100000:0.0:1767318648.776181:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 419 00000100:00100000:0.0:1767318648.776187:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345147f5050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982614784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.776219:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345147f5050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982614784:12345-0@lo:400:kworker.0 Request processed in 32us (94us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318648.776226:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 419 00000100:00080000:0.0:1767318648.776428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318648.776445:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982614784:0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.776483:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982614912:0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.776507:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318648.776529:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982614912 00000100:00100000:0.0:1767318648.776533:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982614912 00000100:00100000:0.0:1767318648.776555:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 420 00000100:00100000:0.0:1767318648.776561:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982614912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318648.776592:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982614912:12345-0@lo:400:kworker.0 Request processed in 31us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318648.776598:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 420 00000100:00080000:0.0:1767318648.776610:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318648.776625:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982614912:0@lo:400:kworker.0 00000100:00100000:3.0:1767318648.778196:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982614144:12345-0@lo:400:kworker.0 Request processed in 3607us (3825us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318648.778209:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 641 00000100:00080000:1.0:1767318648.778297:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318648.778303:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318648.778306:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318648.778311:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3892 00000100:00080000:1.0:1767318648.778428:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318648.778434:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318648.778479:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163982615040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318648.778556:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982615040:0@lo:38:kworker.0 00000100:00100000:0.0:1767318648.778605:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318648.778705:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615040 00000100:00100000:3.0:1767318648.778711:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615040 00000100:00100000:3.0:1767318648.778749:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836 00000100:00100000:3.0:1767318648.778755:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982615040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318648.778773:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:2.0:1767318648.784413:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965f80 x1853163982614144/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318664 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318648.784434:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982614144:0@lo:400:kworker.0 00000100:00100000:3.0:1767318648.786166:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982615040:12345-0@lo:38:kworker.0 Request processed in 7402us (7551us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318648.786181:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836 00000100:00080000:0.0:1767318648.786543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318648.786550:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318648.786554:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982615040:0@lo:38:kworker.0 00010000:00010000:3.0:1767318648.900728:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318649.924557:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318650.309117:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982615168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318650.309193:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318650.309308:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615168 00000100:00100000:2.0:1767318650.309318:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615168 00000100:00100000:2.0:1767318650.309372:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 450 00000100:00100000:2.0:1767318650.309395:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982615168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318650.309438:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318650.309486:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982615168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318650.309494:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 450 00000100:00100000:2.0:1767318650.309553:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982615168:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318650.310143:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318650.310197:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318650.310209:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982615296:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318650.310247:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318650.310706:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615296 00000100:00100000:0.0:1767318650.310713:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615296 00000100:00100000:0.0:1767318650.310770:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 451 00000100:00100000:0.0:1767318650.310778:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982615296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318650.310801:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318650.310845:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982615296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (597us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318650.310861:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 451 00000100:00080000:1.0:1767318650.311703:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318650.311774:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982615296:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318650.767404:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318650.767462:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318650.767557:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580544 00000100:00100000:1.0:1767318650.767570:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580544 00000100:00100000:1.0:1767318650.767654:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580416 00000100:00100000:1.0:1767318650.767658:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580416 00000100:00100000:1.0:1767318650.767676:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 643 00000100:00100000:1.0:1767318650.767687:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915580544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318650.767769:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915580544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (377us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318650.767777:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 643 00000100:00100000:1.0:1767318650.767794:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 837 00000100:00100000:1.0:1767318650.767799:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915580416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318650.767819:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318650.767845:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915580416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (382us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318650.767863:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 837 00000100:00080000:2.0:1767318650.768112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318650.948525:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318651.972544:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318652.997197:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318653.892839:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318653.892848:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318653.892910:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318653.892914:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318653.892919:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318653.892928:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3897 00000100:00080000:3.0:1767318653.892933:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318653.892940:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318653.892945:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318653.892959:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982615424:0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.892961:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163982615552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318653.893014:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318653.893018:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318653.893020:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318653.893023:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3897 00000100:00080000:3.0:1767318653.893026:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318653.893029:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318653.893035:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318653.893048:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1c00 x1853163982615808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318653.893057:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318653.893060:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318653.893062:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318653.893065:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3897 00000100:00080000:3.0:1767318653.893067:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318653.893070:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318653.893083:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2680 x1853163982615936/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318653.893108:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318653.893111:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318653.893113:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318653.893116:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3897 00000100:00080000:3.0:1767318653.893118:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318653.893121:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318653.893122:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982616064:0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.893131:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3b80 x1853163982616320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318653.893149:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318653.893158:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982616192:0@lo:400:kworker.0 00000100:00100000:2.0:1767318653.893170:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318653.893175:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982615680:0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.893183:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318653.893195:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318653.893218:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615424 00000100:00100000:3.0:1767318653.893218:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616064 00000100:00100000:2.0:1767318653.893225:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615424 00000100:00100000:3.0:1767318653.893225:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616064 00000100:00100000:2.0:1767318653.893296:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615680 00000100:00100000:2.0:1767318653.893300:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615680 00000100:00100000:0.0:1767318653.893347:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 644 00000100:00100000:0.0:1767318653.893368:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982615424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318653.893420:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982615424:12345-0@lo:400:kworker.0 Request processed in 65us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318653.893427:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 644 00000100:00100000:0.0:1767318653.893447:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616192 00000100:00100000:0.0:1767318653.893450:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616192 00000100:00100000:3.0:1767318653.894693:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 421 00000100:00100000:3.0:1767318653.894709:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982616064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.894782:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982616064:12345-0@lo:400:kworker.0 Request processed in 79us (1631us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318653.894789:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 421 00000100:00100000:3.0:1767318653.894850:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982616064:0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.894889:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982615552:0@lo:38:kworker.0 00000100:00100000:3.0:1767318653.894924:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318653.894955:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982615808:0@lo:38:kworker.0 00000100:00100000:3.0:1767318653.894975:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318653.895076:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982615936:0@lo:38:kworker.0 00000100:00100000:3.0:1767318653.895100:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318653.895124:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982616320:0@lo:38:kworker.0 00000100:00100000:3.0:1767318653.895149:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767318653.895188:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3480 x1853163982615424/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318669 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318653.895199:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982615424:0@lo:400:kworker.0 00000100:00100000:3.0:1767318653.895229:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615936 00000100:00100000:3.0:1767318653.895234:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615936 00000100:00100000:3.0:1767318653.895255:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 235 00000100:00100000:3.0:1767318653.895260:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982615936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318653.895303:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318653.895339:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982615936:12345-0@lo:38:kworker.0 Request processed in 77us (237us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318653.895344:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 235 00000100:00100000:3.0:1767318653.895433:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615552 00000100:00100000:3.0:1767318653.895437:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615552 00000100:00100000:3.0:1767318653.895450:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982615808 00000100:00100000:3.0:1767318653.895452:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982615808 00000100:00100000:3.0:1767318653.895458:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616320 00000100:00100000:3.0:1767318653.895460:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616320 00000100:00100000:3.0:1767318653.895469:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 838 00000100:00100000:3.0:1767318653.895473:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982615552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318653.895482:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318653.895507:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982615552:12345-0@lo:38:kworker.0 Request processed in 33us (583us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318653.895513:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 838 00000100:00100000:3.0:1767318653.895524:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839 00000100:00100000:3.0:1767318653.895528:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982615808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318653.895536:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318653.895559:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982615808:12345-0@lo:38:kworker.0 Request processed in 29us (582us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318653.895564:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839 00000100:00100000:3.0:1767318653.895575:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 840 00000100:00100000:3.0:1767318653.895578:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982616320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318653.895587:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318653.895715:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982616320:12345-0@lo:38:kworker.0 Request processed in 134us (564us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318653.895721:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 840 00000100:00080000:3.0:1767318653.895748:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318653.895753:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318653.895756:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982615552:0@lo:38:kworker.0 00000100:00080000:3.0:1767318653.895764:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318653.895767:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318653.895770:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982615808:0@lo:38:kworker.0 00000100:00080000:3.0:1767318653.895777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318653.895781:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318653.895784:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982615936:0@lo:38:kworker.0 00000100:00080000:3.0:1767318653.895791:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318653.895794:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318653.895797:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982616320:0@lo:38:kworker.0 00000100:00100000:0.0:1767318653.896006:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 422 00000100:00100000:0.0:1767318653.896015:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982616192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318653.896061:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982616192:12345-0@lo:400:kworker.0 Request processed in 48us (2876us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318653.896066:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 422 00000100:00100000:0.0:1767318653.896095:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982616192:0@lo:400:kworker.0 00000100:00080000:1.0:1767318653.896396:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318653.896402:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318653.896405:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318653.896882:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 645 00000100:00100000:2.0:1767318653.896894:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982615680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318653.896968:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982615680:12345-0@lo:400:kworker.0 Request processed in 77us (3771us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318653.896976:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 645 00010000:00080000:0.0:1767318653.897042:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3800 x1853163982615680/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318669 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318653.897057:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982615680:0@lo:400:kworker.0 00010000:00010000:1.0:1767318654.024261:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318655.044592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318655.428953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982616448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318655.429023:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318655.429250:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616448 00000100:00100000:2.0:1767318655.429259:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616448 00000100:00100000:2.0:1767318655.429355:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 452 00000100:00100000:2.0:1767318655.429366:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982616448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318655.429735:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767318655.429796:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318655.429799:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982616448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 435us (775us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318655.429807:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 452 00000100:00100000:3.0:1767318655.429924:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982616448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318655.431342:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982616576:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318655.431401:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318655.431494:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616576 00000100:00100000:2.0:1767318655.431502:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616576 00000100:00100000:2.0:1767318655.431564:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 453 00000100:00100000:2.0:1767318655.431572:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982616576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318655.431598:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767318655.431644:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318655.449221:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982616576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 16126us (16297us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318655.449238:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 453 00000100:00100000:1.0:1767318655.450229:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982616576:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318655.884089:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318655.884565:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580800 00000100:00100000:2.0:1767318655.884576:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580800 00000100:00100000:2.0:1767318655.884664:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 646 00000100:00100000:2.0:1767318655.884676:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915580800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318655.884794:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915580800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 120us (717us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318655.884803:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 646 00000100:00080000:3.0:1767318655.885354:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318655.888246:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318655.894124:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915580672 00000100:00100000:1.0:1767318655.894140:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915580672 00000100:00100000:1.0:1767318655.894206:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 841 00000100:00100000:1.0:1767318655.894215:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915580672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318655.894257:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318655.894849:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915580672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 629us (6613us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318655.894859:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 841 00010000:00010000:1.0:1767318656.070797:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318657.092805:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318658.128493:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318659.015435:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318659.015445:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318659.015515:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318659.015520:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318659.015524:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318659.015532:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3902 00000100:00080000:0.0:1767318659.015537:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318659.015543:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318659.015570:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247480 x1853163982616832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318659.015607:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318659.015611:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318659.015612:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318659.015615:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3902 00000100:00080000:0.0:1767318659.015618:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318659.015622:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318659.015634:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e245880 x1853163982617088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318659.015642:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318659.015645:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318659.015646:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318659.015649:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3902 00000100:00080000:0.0:1767318659.015652:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318659.015655:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318659.015664:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e244380 x1853163982617216/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318659.015684:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318659.015687:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318659.015689:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318659.015692:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3902 00000100:00080000:0.0:1767318659.015694:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318659.015697:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318659.015825:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982617472:0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.015889:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318659.015951:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318659.015958:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982616704:0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.015983:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318659.016019:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616704 00000100:00100000:1.0:1767318659.016026:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616704 00000100:00100000:1.0:1767318659.016075:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 647 00000100:00100000:1.0:1767318659.016100:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982616704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318659.016149:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982616704:12345-0@lo:400:kworker.0 Request processed in 52us (165us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.016157:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 647 00010000:00080000:1.0:1767318659.016196:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e245500 x1853163982616704/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318675 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318659.016208:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982616704:0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016376:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982616832:0@lo:38:kworker.0 00000100:00100000:3.0:1767318659.016406:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318659.016433:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982617088:0@lo:38:kworker.0 00000100:00100000:3.0:1767318659.016452:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318659.016467:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982617216:0@lo:38:kworker.0 00000100:00100000:3.0:1767318659.016484:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318659.016509:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982616960:0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016526:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318659.016539:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318659.016544:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982617344:0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016560:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318659.016583:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617472 00000100:00100000:3.0:1767318659.016587:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617472 00000100:00100000:3.0:1767318659.016609:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617344 00000100:00100000:3.0:1767318659.016611:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617344 00000100:00100000:3.0:1767318659.016625:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 423 00000100:00100000:3.0:1767318659.016631:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982617472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016658:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982617472:12345-0@lo:400:kworker.0 Request processed in 28us (770us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318659.016664:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 423 00000100:00100000:3.0:1767318659.016674:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 424 00000100:00100000:3.0:1767318659.016679:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982617344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016779:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982617344:12345-0@lo:400:kworker.0 Request processed in 22us (141us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318659.016786:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 424 00000100:00080000:3.0:1767318659.016802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318659.016805:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318659.016808:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318659.016816:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616832 00000100:00100000:3.0:1767318659.016819:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616832 00000100:00100000:3.0:1767318659.016834:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617088 00000100:00100000:3.0:1767318659.016836:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617088 00000100:00100000:3.0:1767318659.016842:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982616960 00000100:00100000:3.0:1767318659.016843:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982616960 00000100:00100000:3.0:1767318659.016861:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 648 00000100:00100000:3.0:1767318659.016866:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982616960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318659.016890:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982616960:12345-0@lo:400:kworker.0 Request processed in 24us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318659.016895:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 648 00000100:00100000:3.0:1767318659.016905:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842 00000100:00100000:3.0:1767318659.016909:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982616832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318659.016924:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318659.016948:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982616832:12345-0@lo:38:kworker.0 Request processed in 37us (542us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318659.016953:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842 00000100:00100000:3.0:1767318659.016962:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 843 00000100:00100000:3.0:1767318659.016965:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982617088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318659.016973:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318659.016988:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982617088:12345-0@lo:38:kworker.0 Request processed in 22us (535us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318659.016993:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 843 00000100:00100000:3.0:1767318659.017005:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617216 00000100:00100000:3.0:1767318659.017008:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617216 00000100:00100000:3.0:1767318659.017022:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 236 00000100:00100000:3.0:1767318659.017025:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982617216:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318659.017034:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318659.017056:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982617216:12345-0@lo:38:kworker.0 Request processed in 30us (571us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318659.017065:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 236 00000100:00100000:1.0:1767318659.017197:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982617472:0@lo:400:kworker.0 00000100:00100000:0.0:1767318659.018855:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ead80 x1853163982617600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318659.029355:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982617344:0@lo:400:kworker.0 00010000:00080000:2.0:1767318659.029377:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e246300 x1853163982616960/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318675 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318659.029389:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982616960:0@lo:400:kworker.0 00000100:00100000:2.0:1767318659.029429:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982617600:0@lo:38:kworker.0 00000100:00100000:2.0:1767318659.029498:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318659.029538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318659.029544:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318659.029547:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982616832:0@lo:38:kworker.0 00000100:00080000:2.0:1767318659.029558:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318659.029561:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318659.029564:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982617088:0@lo:38:kworker.0 00000100:00080000:2.0:1767318659.029574:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318659.029578:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318659.029581:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982617216:0@lo:38:kworker.0 00000100:00080000:2.0:1767318659.029612:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318659.029628:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617600 00000100:00100000:2.0:1767318659.029634:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617600 00000100:00100000:2.0:1767318659.029672:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 844 00000100:00100000:2.0:1767318659.029680:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982617600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318659.031281:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318659.031369:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982617600:12345-0@lo:38:kworker.0 Request processed in 1684us (1867us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318659.031380:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 844 00000100:00080000:1.0:1767318659.031547:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318659.031554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318659.031558:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982617600:0@lo:38:kworker.0 00010000:00010000:1.0:1767318659.140541:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767318659.983420:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915580928 offset 0 length 520: 4 00000100:00100000:0.0:1767318659.984777:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318659.984940:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581056 00000100:00100000:2.0:1767318659.984953:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581056 00000100:00100000:2.0:1767318659.985018:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 845 00000100:00100000:2.0:1767318659.985038:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915581056:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318659.985066:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318659.985151:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915581056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 113us (378us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318659.985159:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 845 00000100:00100000:2.0:1767318659.986718:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318659.986836:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581184 00000100:00100000:1.0:1767318659.986849:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581184 00000100:00100000:1.0:1767318659.986982:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 649 00000100:00100000:1.0:1767318659.986993:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915581184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318659.987072:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915581184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318659.987088:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 649 00000100:00080000:2.0:1767318659.989287:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318660.164432:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318660.548667:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982617728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318660.548750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318660.548837:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617728 00000100:00100000:0.0:1767318660.548846:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617728 00000100:00100000:0.0:1767318660.548898:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 454 00000100:00100000:0.0:1767318660.548906:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982617728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318660.548936:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318660.548986:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982617728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (235us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318660.548992:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 454 00000100:00100000:1.0:1767318660.549084:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982617728:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318660.549519:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982617856:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318660.549550:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318660.549674:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982617856 00000100:00100000:0.0:1767318660.549678:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982617856 00000100:00100000:0.0:1767318660.549734:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 455 00000100:00100000:0.0:1767318660.549740:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982617856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318660.549764:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318660.549906:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982617856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 165us (354us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318660.549913:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 455 00000100:00100000:3.0:1767318660.549961:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982617856:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767318661.188647:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318662.212813:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318663.108474:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318638/real 1767318638] req@ffff93463c625180 x1853163982611456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318663 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318663.108513:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c625180 x1853163982611456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318663 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318663.108536:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318663.108566:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318663.108570:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982611456:0@lo:250:kworker.0 00000100:00080000:1.0:1767318663.108905:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.108911:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318663.108914:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.108919:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3887 00000100:00080000:1.0:1767318663.108923:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.108930:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318663.108966:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982617984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.109576:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.109581:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.109583:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.109587:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.109590:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.109594:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318663.109613:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163982618240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.109639:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.109642:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.109644:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.109648:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.109650:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.109653:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318663.109664:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163982618496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.109672:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.109676:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.109677:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.109681:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.109683:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.109686:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318663.109699:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965c00 x1853163982618624/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318663.109719:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318663.109723:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318663.109725:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318663.109728:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3907 00000100:00080000:1.0:1767318663.109730:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318663.109734:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318663.109744:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964000 x1853163982619008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318663.109787:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982618752:0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.109851:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318663.109914:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318663.109921:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982618368:0@lo:400:kworker.0 00000100:00100000:1.0:1767318663.109952:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318663.110247:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982618112:0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110312:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318663.110335:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318663.110341:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982618880:0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110362:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318663.110392:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618752 00000100:00100000:3.0:1767318663.110398:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618752 00000100:00100000:3.0:1767318663.110446:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618880 00000100:00100000:3.0:1767318663.110450:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618880 00000100:00100000:3.0:1767318663.110472:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 425 00000100:00100000:3.0:1767318663.110481:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982618752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110537:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982618752:12345-0@lo:400:kworker.0 Request processed in 55us (685us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318663.110544:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 425 00000100:00100000:3.0:1767318663.110560:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 426 00000100:00100000:3.0:1767318663.110567:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982618880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110598:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982618880:12345-0@lo:400:kworker.0 Request processed in 32us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318663.110604:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 426 00000100:00100000:3.0:1767318663.110647:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982618880:0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110676:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982618752:0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110697:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618368 00000100:00100000:3.0:1767318663.110701:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618368 00000100:00100000:3.0:1767318663.110726:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618112 00000100:00100000:3.0:1767318663.110729:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618112 00000100:00100000:3.0:1767318663.110749:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 650 00000100:00100000:3.0:1767318663.110755:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982618368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110788:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982618368:12345-0@lo:400:kworker.0 Request processed in 33us (835us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318663.110794:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 650 00000100:00100000:3.0:1767318663.110805:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 651 00000100:00100000:3.0:1767318663.110810:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982618112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318663.110839:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982618112:12345-0@lo:400:kworker.0 Request processed in 28us (527us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318663.110844:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 651 00010000:00080000:3.0:1767318663.110875:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964a80 x1853163982618112/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318679 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318663.110884:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982618112:0@lo:400:kworker.0 00010000:00080000:3.0:1767318663.110904:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964e00 x1853163982618368/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318679 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318663.110913:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982618368:0@lo:400:kworker.0 00000100:00080000:2.0:1767318663.111137:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318663.111143:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318663.111145:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318663.111158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982617984:0@lo:250:kworker.0 00000400:00000100:2.0:1767318663.111194:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982617984 offset 0 length 520: 4 00000100:00100000:2.0:1767318663.111215:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982618240:0@lo:38:kworker.0 00000100:00100000:2.0:1767318663.111238:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318663.111261:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982618496:0@lo:38:kworker.0 00000100:00100000:2.0:1767318663.111309:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318663.111334:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982618624:0@lo:38:kworker.0 00000100:00100000:2.0:1767318663.111357:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318663.111375:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982619008:0@lo:38:kworker.0 00000100:00100000:2.0:1767318663.111395:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318663.111420:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618624 00000100:00100000:2.0:1767318663.111424:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618624 00000100:00100000:2.0:1767318663.111447:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 237 00000100:00100000:2.0:1767318663.111451:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982618624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318663.111469:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318663.111502:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982618624:12345-0@lo:38:kworker.0 Request processed in 49us (143us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318663.111508:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 237 00000100:00080000:2.0:1767318663.111535:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318663.111539:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318663.111542:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982618624:0@lo:38:kworker.0 00000100:00100000:0.0:1767318663.111610:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618240 00000100:00100000:0.0:1767318663.111615:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618240 00000100:00100000:0.0:1767318663.111640:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982618496 00000100:00100000:0.0:1767318663.111644:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982618496 00000100:00100000:0.0:1767318663.111652:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619008 00000100:00100000:0.0:1767318663.111654:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619008 00000100:00100000:0.0:1767318663.111664:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 846 00000100:00100000:0.0:1767318663.111667:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982618240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318663.111677:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318663.111709:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982618240:12345-0@lo:38:kworker.0 Request processed in 40us (469us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318663.111714:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 846 00000100:00100000:0.0:1767318663.111724:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 847 00000100:00100000:0.0:1767318663.111728:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d4450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982618496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318663.111735:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318663.111757:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d4450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982618496:12345-0@lo:38:kworker.0 Request processed in 28us (448us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318663.111762:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 847 00000100:00100000:3.0:1767318663.112484:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 848 00000100:00100000:3.0:1767318663.112491:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982619008:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318663.112501:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318663.112531:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982619008:12345-0@lo:38:kworker.0 Request processed in 40us (1135us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318663.112537:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 848 00000100:00080000:2.0:1767318663.112830:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318663.112835:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318663.112838:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982618240:0@lo:38:kworker.0 00000100:00080000:2.0:1767318663.112848:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318663.112851:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318663.112854:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982618496:0@lo:38:kworker.0 00000100:00080000:2.0:1767318663.112877:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318663.112885:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318663.112888:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982619008:0@lo:38:kworker.0 00010000:00010000:1.0:1767318663.236476:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318664.261862:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318665.100749:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318665.102360:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318665.113667:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581312 00000100:00100000:1.0:1767318665.113684:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581312 00000100:00100000:3.0:1767318665.116491:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581440 00000100:00100000:3.0:1767318665.116504:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581440 00000100:00100000:3.0:1767318665.116616:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 652 00000100:00100000:3.0:1767318665.116627:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915581440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318665.116719:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915581440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (14368us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318665.116728:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 652 00000100:00100000:3.0:1767318665.116748:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 849 00000100:00100000:3.0:1767318665.116754:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318665.116775:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318665.116801:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (16064us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318665.116808:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 849 00000100:00080000:2.0:1767318665.117900:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318665.286096:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318665.668881:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982619136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318665.668956:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318665.670767:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619136 00000100:00100000:2.0:1767318665.670780:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619136 00000100:00100000:2.0:1767318665.670861:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 456 00000100:00100000:2.0:1767318665.670871:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982619136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318665.670903:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318665.670973:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982619136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (2015us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318665.670981:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 456 00000100:00080000:0.0:1767318665.671410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318665.671720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982619136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318665.672756:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982619264:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318665.672810:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318665.673482:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619264 00000100:00100000:2.0:1767318665.673490:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619264 00000100:00100000:2.0:1767318665.673565:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 457 00000100:00100000:2.0:1767318665.673575:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982619264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318665.673599:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318665.673652:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982619264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (841us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318665.673659:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 457 00000100:00100000:1.0:1767318665.673741:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982619264:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318665.673869:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318666.308530:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318667.334932:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318668.235130:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318668.235141:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318668.235244:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767318668.235266:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318668.235291:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318668.235296:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318668.235304:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3911 00000100:00080000:2.0:1767318668.235309:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318668.235316:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318668.235338:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163982619520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318668.235374:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318668.235378:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318668.235380:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318668.235383:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3911 00000100:00080000:2.0:1767318668.235386:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318668.235389:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318668.235400:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581500 x1853163982619776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318668.235409:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318668.235413:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318668.235414:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318668.235418:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3911 00000100:00080000:2.0:1767318668.235420:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318668.235424:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318668.235435:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582a00 x1853163982619904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318668.235577:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982619520:0@lo:38:kworker.0 00000100:00100000:0.0:1767318668.235648:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318668.235682:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982619776:0@lo:38:kworker.0 00000100:00100000:0.0:1767318668.235704:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318668.235721:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982619904:0@lo:38:kworker.0 00000100:00100000:0.0:1767318668.235744:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318668.235781:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619520 00000100:00100000:0.0:1767318668.235788:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619520 00000100:00100000:0.0:1767318668.235816:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619776 00000100:00100000:0.0:1767318668.235819:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619776 00000100:00100000:0.0:1767318668.235836:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 850 00000100:00100000:0.0:1767318668.235843:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982619520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318668.235863:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318668.235902:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982619520:12345-0@lo:38:kworker.0 Request processed in 57us (254us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318668.235909:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 850 00000100:00100000:0.0:1767318668.235925:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 851 00000100:00100000:0.0:1767318668.235930:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982619776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318668.235939:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318668.235971:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982619776:12345-0@lo:38:kworker.0 Request processed in 40us (267us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318668.235981:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 851 00000100:00080000:0.0:1767318668.236013:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318668.236018:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318668.236022:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982619520:0@lo:38:kworker.0 00000100:00080000:0.0:1767318668.236378:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318668.236383:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318668.236386:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982619776:0@lo:38:kworker.0 00000100:00100000:0.0:1767318668.236426:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982620032:0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.237218:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318668.237232:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982620160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.237341:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982619648:0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.237367:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318668.237404:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619648 00000100:00100000:3.0:1767318668.237410:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619648 00000100:00100000:3.0:1767318668.237454:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 653 00000100:00100000:3.0:1767318668.237462:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982619648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.237505:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982619648:12345-0@lo:400:kworker.0 Request processed in 44us (136us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318668.237613:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 653 00000100:00080000:3.0:1767318668.237634:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767318668.237656:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580700 x1853163982619648/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318684 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318668.237667:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982619648:0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.237695:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619904 00000100:00100000:3.0:1767318668.237700:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619904 00000100:00100000:3.0:1767318668.237720:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 238 00000100:00100000:3.0:1767318668.237725:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982619904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318668.237739:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318668.237821:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982619904:12345-0@lo:38:kworker.0 Request processed in 94us (2075us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318668.237828:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 238 00000100:00080000:3.0:1767318668.237874:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318668.237878:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318668.237881:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982619904:0@lo:38:kworker.0 00000100:00100000:1.0:1767318668.238102:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982619392:0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.238133:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318668.238170:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982619392 00000100:00100000:1.0:1767318668.238174:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982619392 00000100:00100000:1.0:1767318668.238201:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 654 00000100:00100000:1.0:1767318668.238207:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982619392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.238239:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982619392:12345-0@lo:400:kworker.0 Request processed in 33us (105us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318668.238245:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 654 00010000:00080000:1.0:1767318668.238281:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581880 x1853163982619392/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318684 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318668.238292:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982619392:0@lo:400:kworker.0 00000100:00080000:3.0:1767318668.238362:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318668.238823:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318668.238828:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318668.238832:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767318668.238834:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318668.238836:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3911 00000100:00080000:2.0:1767318668.238840:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318668.238844:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318668.238922:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620032 00000100:00100000:1.0:1767318668.238926:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620032 00000100:00080000:2.0:1767318668.238972:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318668.238977:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 427 00000100:00100000:1.0:1767318668.238984:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982620032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318668.239028:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982620032:12345-0@lo:400:kworker.0 Request processed in 44us (2570us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318668.239034:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 427 00000100:00100000:2.0:1767318668.239055:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982620032:0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.239145:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163982620288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318668.239183:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318668.239197:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982620288:0@lo:38:kworker.0 00000100:00100000:0.0:1767318668.239226:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318668.239291:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620160 00000100:00100000:2.0:1767318668.239291:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620288 00000100:00100000:1.0:1767318668.239295:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620160 00000100:00100000:2.0:1767318668.239295:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620288 00000100:00100000:2.0:1767318668.239355:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 852 00000100:00100000:2.0:1767318668.239360:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982620288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318668.239375:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318668.239409:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982620288:12345-0@lo:38:kworker.0 Request processed in 47us (181us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318668.239416:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 852 00000100:00080000:1.0:1767318668.239428:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318668.239437:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318668.239440:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982620288:0@lo:38:kworker.0 00000100:00100000:3.0:1767318668.239679:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 428 00000100:00100000:3.0:1767318668.239687:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982620160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318668.239731:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982620160:12345-0@lo:400:kworker.0 Request processed in 45us (547us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318668.239737:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 428 00000100:00100000:2.0:1767318668.239798:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982620160:0@lo:400:kworker.0 00000100:00080000:1.0:1767318668.239827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318668.356583:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318669.432544:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318670.224204:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318670.224267:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318670.224393:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581696 00000100:00100000:0.0:1767318670.224405:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581696 00000100:00100000:0.0:1767318670.224510:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581568 00000100:00100000:0.0:1767318670.224514:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581568 00000100:00100000:0.0:1767318670.224538:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 655 00000100:00100000:0.0:1767318670.224550:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915581696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318670.224639:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915581696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (432us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318670.224649:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 655 00000100:00100000:0.0:1767318670.224667:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 853 00000100:00100000:0.0:1767318670.224672:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581568:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318670.224701:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318670.224746:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (471us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318670.224753:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 853 00000100:00080000:1.0:1767318670.224891:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318670.468439:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318670.789786:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318670.789813:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982620416:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318670.789885:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318670.790016:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620416 00000100:00100000:3.0:1767318670.790026:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620416 00000100:00100000:3.0:1767318670.790102:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 458 00000100:00100000:3.0:1767318670.790112:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982620416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318670.790142:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318670.790203:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982620416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318670.790216:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 458 00000100:00080000:2.0:1767318670.791541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318670.791590:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982620416:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318670.791936:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982620544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318670.791986:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318670.792527:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620544 00000100:00100000:0.0:1767318670.792534:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620544 00000100:00100000:0.0:1767318670.792589:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 459 00000100:00100000:0.0:1767318670.792597:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982620544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318670.792624:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318670.792679:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982620544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (692us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318670.792686:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 459 00000100:00100000:0.0:1767318670.792723:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982620544:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318670.793331:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318671.495262:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318672.520227:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318673.348575:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318673.348586:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318673.348649:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318673.348654:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318673.348658:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318673.348665:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3916 00000100:00080000:0.0:1767318673.348669:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318673.348676:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318673.348698:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163982620800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318673.348749:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318673.348753:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318673.348755:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318673.348758:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3916 00000100:00080000:0.0:1767318673.348762:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318673.348766:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318673.348778:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea680 x1853163982621056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318673.348787:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318673.348791:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318673.348792:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318673.348796:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3916 00000100:00080000:0.0:1767318673.348798:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318673.348802:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318673.348813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163982621184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318673.348834:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318673.348838:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318673.348839:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318673.348843:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3916 00000100:00080000:0.0:1767318673.348845:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318673.348848:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318673.348858:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb800 x1853163982621568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318673.348886:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982621312:0@lo:400:kworker.0 00000100:00100000:0.0:1767318673.348950:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318673.349028:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318673.349035:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982620928:0@lo:400:kworker.0 00000100:00100000:2.0:1767318673.349047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982620800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318673.349062:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318673.349142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982620672:0@lo:400:kworker.0 00000100:00100000:3.0:1767318673.349186:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982621440:0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.351897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318673.351999:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620928 00000100:00100000:1.0:1767318673.352009:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620928 00000100:00100000:1.0:1767318673.352067:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620672 00000100:00100000:1.0:1767318673.352070:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620672 00000100:00100000:1.0:1767318673.352111:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767318673.352120:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982620928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.352174:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982620928:12345-0@lo:400:kworker.0 Request processed in 55us (3110us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.352181:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767318673.352199:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 657 00000100:00100000:1.0:1767318673.352206:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982620672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.352238:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982620672:12345-0@lo:400:kworker.0 Request processed in 33us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.352244:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 657 00000100:00080000:1.0:1767318673.352261:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318673.352322:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8700 x1853163982620672/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318689 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318673.352336:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982620672:0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.352380:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621312 00000100:00100000:1.0:1767318673.352384:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621312 00000100:00100000:1.0:1767318673.352415:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 429 00000100:00100000:1.0:1767318673.352422:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982621312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.352460:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982621312:12345-0@lo:400:kworker.0 Request processed in 39us (3510us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.352466:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 429 00000100:00080000:1.0:1767318673.352479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318673.352889:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318673.353021:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767318673.353036:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9880 x1853163982620928/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318689 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318673.353047:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982620928:0@lo:400:kworker.0 00000100:00100000:2.0:1767318673.353050:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982621056:0@lo:38:kworker.0 00000100:00100000:0.0:1767318673.353059:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982621312:0@lo:400:kworker.0 00000100:00100000:2.0:1767318673.353077:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318673.353094:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982621184:0@lo:38:kworker.0 00000100:00100000:2.0:1767318673.353116:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318673.353129:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982620800 00000100:00100000:0.0:1767318673.353133:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982620800 00000100:00100000:2.0:1767318673.353140:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982621568:0@lo:38:kworker.0 00000100:00100000:0.0:1767318673.353151:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621056 00000100:00100000:0.0:1767318673.353154:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621056 00000100:00100000:2.0:1767318673.353159:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318673.353165:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621568 00000100:00100000:0.0:1767318673.353167:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621568 00000100:00100000:0.0:1767318673.353178:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 854 00000100:00100000:0.0:1767318673.353182:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982620800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318673.353201:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318673.353203:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 855 00000100:00100000:2.0:1767318673.353207:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982621056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318673.353216:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318673.353236:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982620800:12345-0@lo:38:kworker.0 Request processed in 52us (214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318673.353236:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982621056:12345-0@lo:38:kworker.0 Request processed in 26us (157us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318673.353242:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 854 00000100:00100000:2.0:1767318673.353244:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 855 00000100:00100000:0.0:1767318673.353253:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 856 00000100:00100000:0.0:1767318673.353257:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982621568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318673.353265:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767318673.353290:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318673.353295:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318673.353298:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982620800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318673.353302:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982621568:12345-0@lo:38:kworker.0 Request processed in 44us (143us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767318673.353307:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318673.353310:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318673.353312:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982621056:0@lo:38:kworker.0 00000100:00080000:2.0:1767318673.353319:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318673.353323:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318673.353325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982621568:0@lo:38:kworker.0 00000100:00100000:3.0:1767318673.353595:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621184 00000100:00100000:3.0:1767318673.353599:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621184 00000100:00100000:3.0:1767318673.353640:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 239 00000100:00100000:3.0:1767318673.353646:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982621184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318673.353656:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318673.353685:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982621184:12345-0@lo:38:kworker.0 Request processed in 38us (567us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318673.353690:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 239 00000100:00100000:0.0:1767318673.353799:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 856 00000100:00080000:2.0:1767318673.353870:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318673.353874:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318673.353877:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982621184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318673.353938:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621440 00000100:00100000:1.0:1767318673.353942:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621440 00000100:00100000:1.0:1767318673.354170:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 430 00000100:00100000:1.0:1767318673.354177:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982621440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318673.354226:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982621440:12345-0@lo:400:kworker.0 Request processed in 50us (1338us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318673.354233:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 430 00000100:00100000:0.0:1767318673.354318:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982621440:0@lo:400:kworker.0 00000100:00080000:2.0:1767318673.354715:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318673.542159:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318674.566051:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318675.339817:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318675.339868:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318675.339966:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581824 00000100:00100000:3.0:1767318675.339977:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581824 00000100:00100000:3.0:1767318675.340025:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915581952 00000100:00100000:3.0:1767318675.340028:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915581952 00000100:00100000:3.0:1767318675.340082:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 658 00000100:00100000:3.0:1767318675.340092:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915581952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767318675.340131:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318675.340155:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 857 00000100:00100000:2.0:1767318675.340160:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581824:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:3.0:1767318675.340175:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915581952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (306us total) trans 0 rc 0/0 00010000:00020000:2.0:1767318675.340180:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318675.340182:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 658 00000100:00100000:2.0:1767318675.340206:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915581824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (401us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318675.340212:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 857 00010000:00010000:1.0:1767318675.588497:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318675.908541:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318675.908567:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982621696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318675.908637:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318675.908921:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621696 00000100:00100000:2.0:1767318675.908931:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621696 00000100:00100000:2.0:1767318675.909026:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 460 00000100:00100000:2.0:1767318675.909036:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982621696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318675.909065:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767318675.909236:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318675.909371:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982621824:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318675.909416:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318675.909503:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621824 00000100:00100000:3.0:1767318675.909508:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621824 00000100:00100000:3.0:1767318675.909542:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 461 00000100:00100000:3.0:1767318675.909549:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982621824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318675.909565:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318675.909605:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982621824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (188us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318675.909612:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 461 00000100:00100000:3.0:1767318675.909660:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982621824:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318675.909719:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318675.909911:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982621696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 876us (1274us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318675.909919:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 460 00000100:00100000:3.0:1767318675.909995:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982621696:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318676.612797:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318677.638260:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318678.471814:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.471832:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318678.471903:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.471907:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.471911:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.471919:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.471925:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.471932:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318678.471957:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163982622080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.471997:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.472000:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.472002:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.472006:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.472008:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.472012:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318678.472022:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163982622336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.472030:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.472033:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.472034:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.472037:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.472039:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.472042:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318678.472051:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163982622464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318678.472068:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318678.472071:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318678.472073:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318678.472076:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3921 00000100:00080000:3.0:1767318678.472077:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318678.472080:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318678.472087:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966300 x1853163982622848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318678.472113:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982622208:0@lo:400:kworker.0 00000100:00100000:3.0:1767318678.472179:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318678.472247:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318678.472252:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982622592:0@lo:400:kworker.0 00000100:00100000:3.0:1767318678.472307:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318678.472452:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982622080:0@lo:38:kworker.0 00000100:00100000:0.0:1767318678.472482:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318678.472505:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982622336:0@lo:38:kworker.0 00000100:00100000:0.0:1767318678.472521:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318678.472534:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982622464:0@lo:38:kworker.0 00000100:00100000:0.0:1767318678.472551:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318678.472565:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982622848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318678.472579:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318678.472606:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622208 00000100:00100000:0.0:1767318678.472613:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622208 00000100:00100000:0.0:1767318678.472658:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622080 00000100:00100000:0.0:1767318678.472660:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622080 00000100:00100000:0.0:1767318678.472679:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622336 00000100:00100000:0.0:1767318678.472682:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622336 00000100:00100000:0.0:1767318678.472688:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622848 00000100:00100000:0.0:1767318678.472690:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622848 00000100:00100000:0.0:1767318678.472701:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 659 00000100:00100000:0.0:1767318678.472709:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982622208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318678.472758:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982622208:12345-0@lo:400:kworker.0 Request processed in 51us (579us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318678.472766:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 659 00000100:00100000:0.0:1767318678.472782:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 858 00000100:00100000:0.0:1767318678.472785:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318678.472801:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318678.472826:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622080:12345-0@lo:38:kworker.0 Request processed in 39us (344us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318678.472832:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 858 00000100:00100000:0.0:1767318678.472841:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 859 00000100:00100000:0.0:1767318678.472844:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318678.472851:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318678.472867:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622336:12345-0@lo:38:kworker.0 Request processed in 22us (345us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318678.472871:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 859 00000100:00100000:0.0:1767318678.472879:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 860 00000100:00100000:0.0:1767318678.472882:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318678.472902:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318678.472920:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982622848:12345-0@lo:38:kworker.0 Request processed in 37us (341us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318678.472925:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 860 00000100:00080000:0.0:1767318678.472956:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318678.472960:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318678.472963:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982622080:0@lo:38:kworker.0 00000100:00080000:0.0:1767318678.472969:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318678.472972:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318678.472974:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982622336:0@lo:38:kworker.0 00000100:00080000:0.0:1767318678.472980:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318678.472983:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318678.472985:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982622848:0@lo:38:kworker.0 00010000:00080000:0.0:1767318678.473014:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964000 x1853163982622208/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318694 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318678.473025:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982622208:0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.473248:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982622720:0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.473303:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318678.473324:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318678.473330:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982621952:0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.473358:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318678.473382:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982621952 00000100:00100000:1.0:1767318678.473385:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982621952 00000100:00100000:1.0:1767318678.473413:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 660 00000100:00100000:1.0:1767318678.473419:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982621952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318678.473447:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982621952:12345-0@lo:400:kworker.0 Request processed in 29us (88us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318678.473453:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 660 00000100:00080000:1.0:1767318678.473468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318678.473485:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0a80 x1853163982621952/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318694 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318678.473493:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982621952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318678.473637:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622464 00000100:00100000:2.0:1767318678.473642:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622464 00000100:00100000:2.0:1767318678.473678:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 240 00000100:00100000:2.0:1767318678.473683:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982622464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318678.473693:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318678.473723:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982622464:12345-0@lo:38:kworker.0 Request processed in 39us (1171us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318678.473728:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 240 00000100:00080000:2.0:1767318678.473762:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318678.473766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318678.473769:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982622464:0@lo:38:kworker.0 00000100:00100000:3.0:1767318678.474026:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622592 00000100:00100000:3.0:1767318678.474040:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622592 00000100:00100000:3.0:1767318678.474073:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622720 00000100:00100000:3.0:1767318678.474076:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622720 00000100:00100000:3.0:1767318678.474094:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 431 00000100:00100000:3.0:1767318678.474100:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982622592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318678.474142:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982622592:12345-0@lo:400:kworker.0 Request processed in 42us (1833us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318678.474148:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 431 00000100:00100000:3.0:1767318678.474160:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 432 00000100:00100000:3.0:1767318678.474166:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982622720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318678.474196:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982622720:12345-0@lo:400:kworker.0 Request processed in 30us (892us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318678.474201:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 432 00000100:00080000:2.0:1767318678.474661:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318678.474675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318678.474712:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982622592:0@lo:400:kworker.0 00000100:00100000:2.0:1767318678.474741:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982622720:0@lo:400:kworker.0 00010000:00010000:1.0:1767318678.661461:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318679.684752:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318680.460721:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318680.460769:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318680.460887:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582208 00000100:00100000:2.0:1767318680.460898:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582208 00000100:00100000:2.0:1767318680.460975:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582080 00000100:00100000:2.0:1767318680.460979:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582080 00000100:00100000:2.0:1767318680.461000:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 661 00000100:00100000:2.0:1767318680.461010:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915582208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318680.461090:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915582208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (381us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318680.461098:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 661 00000100:00100000:2.0:1767318680.461118:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 861 00000100:00100000:2.0:1767318680.461122:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915582080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318680.461142:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318680.461169:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915582080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (400us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318680.461190:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 861 00000100:00080000:1.0:1767318680.461399:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318680.711521:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318681.028486:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1767318681.028626:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982622976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318681.028696:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318681.028807:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982623104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318681.028834:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318681.029438:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982622976 00000100:00100000:3.0:1767318681.029447:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982622976 00000100:00100000:3.0:1767318681.029551:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623104 00000100:00100000:3.0:1767318681.029555:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623104 00000100:00100000:2.0:1767318681.029605:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 462 00000100:00080000:1.0:1767318681.029618:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318681.029622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318681.029625:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982622976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318681.029655:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318681.029710:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982622976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (1014us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318681.029718:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 462 00000100:00100000:2.0:1767318681.029736:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 463 00000100:00100000:2.0:1767318681.029743:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982623104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318681.029759:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318681.029795:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982623104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (961us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318681.029801:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 463 00000100:00100000:0.0:1767318681.029807:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982622976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318681.029822:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982623104:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318681.734589:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318682.757452:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318683.588642:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318683.588654:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318683.588753:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:0.0:1767318683.588772:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318683.588776:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318683.588779:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318683.588798:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3927 00000100:00080000:0.0:1767318683.588803:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318683.588809:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318683.588834:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163982623360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318683.588950:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982623360:0@lo:38:kworker.0 00000100:00100000:2.0:1767318683.589017:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318683.589084:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623360 00000100:00100000:2.0:1767318683.589092:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623360 00000100:00100000:2.0:1767318683.589123:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 862 00000100:00100000:2.0:1767318683.589130:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982623360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318683.589148:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318683.589189:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982623360:12345-0@lo:38:kworker.0 Request processed in 58us (171us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318683.589195:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 862 00000100:00080000:2.0:1767318683.589229:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318683.589234:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318683.589237:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982623360:0@lo:38:kworker.0 00000100:00100000:3.0:1767318683.589330:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318683.589341:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982623488:0@lo:400:kworker.0 00000100:00100000:3.0:1767318683.589370:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318683.589712:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623488 00000100:00100000:2.0:1767318683.589717:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623488 00000100:00100000:2.0:1767318683.589767:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 662 00000100:00100000:2.0:1767318683.589774:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982623488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318683.589830:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982623488:12345-0@lo:400:kworker.0 Request processed in 56us (459us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318683.589837:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 662 00010000:00080000:2.0:1767318683.589868:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8700 x1853163982623488/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318699 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318683.589880:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982623488:0@lo:400:kworker.0 00000100:00080000:3.0:1767318683.589940:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318683.592064:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982623232:0@lo:400:kworker.0 00000100:00100000:1.0:1767318683.592138:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318683.592397:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318683.592403:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318683.592407:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318683.592412:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3927 00000100:00080000:0.0:1767318683.592416:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318683.592422:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318683.592617:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982623616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318683.592647:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318683.592651:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318683.592653:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318683.592656:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3927 00000100:00080000:0.0:1767318683.592659:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318683.592662:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318683.592676:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9500 x1853163982623744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318683.592736:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982623616:0@lo:38:kworker.0 00000100:00100000:3.0:1767318683.592774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318683.592820:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318683.592823:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318683.592825:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318683.592829:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3927 00000100:00080000:0.0:1767318683.592832:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318683.592835:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318683.592846:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0a80 x1853163982624128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318683.592911:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982624000:0@lo:400:kworker.0 00000100:00100000:0.0:1767318683.592942:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318683.592970:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318683.592976:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982623872:0@lo:400:kworker.0 00000100:00100000:0.0:1767318683.592996:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318683.593026:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624000 00000100:00100000:0.0:1767318683.593032:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624000 00000100:00100000:0.0:1767318683.593081:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623872 00000100:00100000:0.0:1767318683.593084:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623872 00000100:00100000:0.0:1767318683.593108:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 433 00000100:00100000:0.0:1767318683.593116:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982624000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318683.593165:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982624000:12345-0@lo:400:kworker.0 Request processed in 49us (221us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318683.593172:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 433 00000100:00100000:0.0:1767318683.593187:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 434 00000100:00100000:0.0:1767318683.593193:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982623872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318683.593219:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982623872:12345-0@lo:400:kworker.0 Request processed in 27us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318683.593224:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 434 00000100:00100000:0.0:1767318683.593255:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982623872:0@lo:400:kworker.0 00000100:00100000:0.0:1767318683.593264:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982624000:0@lo:400:kworker.0 00000100:00080000:0.0:1767318683.593324:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318683.593327:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318683.593582:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982623744:0@lo:38:kworker.0 00000100:00100000:3.0:1767318683.593610:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318683.593634:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982624128:0@lo:38:kworker.0 00000100:00100000:3.0:1767318683.593655:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318683.593726:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623744 00000100:00100000:1.0:1767318683.593731:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623744 00000100:00100000:1.0:1767318683.593757:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 241 00000100:00100000:1.0:1767318683.593762:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982623744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318683.593780:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318683.593827:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982623744:12345-0@lo:38:kworker.0 Request processed in 63us (215us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318683.593834:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 241 00000100:00100000:2.0:1767318683.593882:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623232 00000100:00100000:2.0:1767318683.593887:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623232 00000100:00100000:2.0:1767318683.593963:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982623616 00000100:00100000:2.0:1767318683.593966:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982623616 00000100:00100000:2.0:1767318683.593982:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 663 00000100:00100000:2.0:1767318683.593989:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982623232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318683.594027:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982623232:12345-0@lo:400:kworker.0 Request processed in 39us (1890us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318683.594033:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 663 00000100:00100000:2.0:1767318683.594046:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 863 00000100:00100000:2.0:1767318683.594050:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982623616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318683.594059:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318683.594081:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982623616:12345-0@lo:38:kworker.0 Request processed in 30us (1307us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318683.594087:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 863 00010000:00080000:2.0:1767318683.594117:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ea680 x1853163982623232/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318699 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318683.594128:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982623232:0@lo:400:kworker.0 00000100:00080000:1.0:1767318683.594175:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318683.594235:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318683.594240:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318683.594244:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982623616:0@lo:38:kworker.0 00000100:00080000:3.0:1767318683.594252:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318683.594255:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318683.594258:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982623744:0@lo:38:kworker.0 00000100:00100000:2.0:1767318683.594377:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624128 00000100:00100000:2.0:1767318683.594381:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624128 00000100:00100000:0.0:1767318683.594463:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 864 00000100:00100000:0.0:1767318683.594469:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982624128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318683.594479:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318683.594637:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982624128:12345-0@lo:38:kworker.0 Request processed in 167us (980us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318683.594644:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 864 00000100:00080000:3.0:1767318683.594708:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318683.594713:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318683.594716:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982624128:0@lo:38:kworker.0 00010000:00010000:1.0:1767318683.780461:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318684.804843:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318685.579907:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318685.580878:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582592 00000100:00100000:2.0:1767318685.580892:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582592 00000400:00000100:3.0:1767318685.580939:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915582336 offset 0 length 520: 4 00000100:00100000:2.0:1767318685.581254:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 664 00000100:00100000:2.0:1767318685.581265:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915582592:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318685.581389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318685.581624:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915582592:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 359us (1730us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318685.581632:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 664 00000100:00100000:3.0:1767318685.581909:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318685.582024:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582464 00000100:00100000:2.0:1767318685.582029:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582464 00000100:00100000:2.0:1767318685.582050:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 865 00000100:00100000:2.0:1767318685.582055:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915582464:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318685.582076:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318685.582132:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915582464:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 74us (222us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318685.582139:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 865 00010000:00010000:1.0:1767318685.834429:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318686.149103:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982624256:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318686.149268:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318686.149373:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982624384:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318686.149408:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318686.149881:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624256 00000100:00100000:0.0:1767318686.149890:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624256 00000100:00080000:1.0:1767318686.149956:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318686.149958:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624384 00000100:00100000:0.0:1767318686.149963:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624384 00000100:00100000:1.0:1767318686.149994:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 464 00000100:00100000:1.0:1767318686.150005:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982624256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318686.150033:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318686.150082:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982624256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (814us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318686.150441:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982624256:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318686.150543:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318686.150810:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 465 00000100:00100000:0.0:1767318686.150822:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982624384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318686.150844:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318686.150935:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982624384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 116us (1526us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318686.150942:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 465 00000100:00100000:2.0:1767318686.151018:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982624384:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318686.154048:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 464 00010000:00010000:1.0:1767318686.857379:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318687.876776:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318688.708442:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318663/real 1767318663] req@ffff93463f967480 x1853163982617984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318688 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318688.708489:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463f967480 x1853163982617984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318688 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318688.708511:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.708534:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318688.708539:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982617984:0@lo:250:kworker.0 00000100:00080000:0.0:1767318688.709020:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318688.709026:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318688.709029:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318688.709035:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3911 00000100:00080000:0.0:1767318688.709039:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318688.709046:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318688.709087:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163982624512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318688.709134:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318688.709138:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318688.709140:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318688.709144:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3932 00000100:00080000:0.0:1767318688.709147:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318688.709151:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318688.709169:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ed80 x1853163982624768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318688.709173:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982624640:0@lo:400:kworker.0 00000100:00080000:0.0:1767318688.709192:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318688.709195:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318688.709198:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318688.709203:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3932 00000100:00080000:0.0:1767318688.709205:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318688.709210:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318688.709222:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f100 x1853163982625024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318688.709231:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318688.709234:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318688.709235:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318688.709239:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3932 00000100:00080000:0.0:1767318688.709241:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318688.709245:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318688.709257:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c700 x1853163982625152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318688.709265:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318688.709350:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318688.709356:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982624896:0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.709394:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318688.709410:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982625280:0@lo:400:kworker.0 00000100:00080000:0.0:1767318688.709553:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318688.709557:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318688.709559:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318688.709562:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3932 00000100:00080000:0.0:1767318688.709805:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318688.709810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318688.709828:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543df80 x1853163982625536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318688.709861:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624640 00000100:00100000:0.0:1767318688.709868:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624640 00000100:00100000:0.0:1767318688.709926:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624896 00000100:00100000:0.0:1767318688.709931:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624896 00000100:00100000:0.0:1767318688.709957:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 665 00000100:00100000:0.0:1767318688.709965:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982624640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318688.710020:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982624512:0@lo:250:kworker.0 00000100:00100000:2.0:1767318688.715416:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318688.715494:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318688.715507:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982625408:0@lo:400:kworker.0 00000100:00100000:2.0:1767318688.715547:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318688.715584:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625280 00000100:00100000:2.0:1767318688.715591:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625280 00000100:00100000:2.0:1767318688.715645:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625408 00000100:00100000:1.0:1767318688.715650:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 666 00000100:00100000:2.0:1767318688.715650:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625408 00000100:00100000:1.0:1767318688.715667:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982624896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318688.715678:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 435 00000100:00100000:2.0:1767318688.715687:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982625280:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318688.715752:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982624896:12345-0@lo:400:kworker.0 Request processed in 90us (6357us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318688.715754:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982625280:12345-0@lo:400:kworker.0 Request processed in 68us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318688.715765:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 666 00000100:00100000:2.0:1767318688.715766:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 435 00000100:00100000:2.0:1767318688.715788:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 436 00000100:00100000:2.0:1767318688.715796:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982625408:12345-0@lo:400:kworker.0 00010000:00080000:1.0:1767318688.715818:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543ca80 x1853163982624896/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318704 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318688.715828:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982625408:12345-0@lo:400:kworker.0 Request processed in 32us (278us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318688.715836:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 436 00000100:00100000:1.0:1767318688.715838:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982624896:0@lo:400:kworker.0 00000100:00080000:2.0:1767318688.715877:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318688.715882:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318688.715884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318688.715910:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982625408:0@lo:400:kworker.0 00000100:00100000:2.0:1767318688.715926:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982625280:0@lo:400:kworker.0 00000100:00100000:0.0:1767318688.715938:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982624640:12345-0@lo:400:kworker.0 Request processed in 5970us (6674us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318688.715949:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 665 00010000:00080000:1.0:1767318688.715954:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543fb80 x1853163982624640/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318704 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318688.715965:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982624640:0@lo:400:kworker.0 00000400:00000100:3.0:1767318688.716249:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982624512 offset 0 length 520: 4 00000100:00100000:3.0:1767318688.716305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982624768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318688.716715:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318688.716746:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982625024:0@lo:38:kworker.0 00000100:00100000:3.0:1767318688.716768:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318688.716796:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982625152:0@lo:38:kworker.0 00000100:00100000:3.0:1767318688.716823:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318688.716853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982625536:0@lo:38:kworker.0 00000100:00100000:3.0:1767318688.716875:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318688.716914:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982624768 00000100:00100000:3.0:1767318688.716920:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982624768 00000100:00100000:3.0:1767318688.716946:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625024 00000100:00100000:3.0:1767318688.716948:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625024 00000100:00100000:3.0:1767318688.716960:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625536 00000100:00100000:3.0:1767318688.716963:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625536 00000100:00100000:3.0:1767318688.716973:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 866 00000100:00100000:3.0:1767318688.716979:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982624768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318688.717000:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318688.717037:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982624768:12345-0@lo:38:kworker.0 Request processed in 57us (322us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318688.717044:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 866 00000100:00100000:3.0:1767318688.717058:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 867 00000100:00100000:3.0:1767318688.717062:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982625024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318688.717071:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318688.717093:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982625024:12345-0@lo:38:kworker.0 Request processed in 30us (324us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318688.717099:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 867 00000100:00100000:3.0:1767318688.717108:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 868 00000100:00100000:3.0:1767318688.717112:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982625536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318688.717121:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318688.717142:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982625536:12345-0@lo:38:kworker.0 Request processed in 28us (266us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318688.717147:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 868 00000100:00080000:3.0:1767318688.717181:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.717187:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318688.717190:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982624768:0@lo:38:kworker.0 00000100:00080000:3.0:1767318688.717198:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.717202:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318688.717205:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982625024:0@lo:38:kworker.0 00000100:00080000:3.0:1767318688.717212:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.717216:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318688.717219:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982625536:0@lo:38:kworker.0 00000100:00100000:2.0:1767318688.718453:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625152 00000100:00100000:2.0:1767318688.718463:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625152 00000100:00100000:2.0:1767318688.718521:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 242 00000100:00100000:2.0:1767318688.718528:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982625152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318688.718543:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318688.718608:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982625152:12345-0@lo:38:kworker.0 Request processed in 80us (1784us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318688.718615:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 242 00000100:00080000:3.0:1767318688.718659:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318688.718664:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318688.718667:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982625152:0@lo:38:kworker.0 00010000:00010000:1.0:1767318688.901061:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318689.924517:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318690.699900:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318690.699944:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318690.699971:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582720 00000100:00100000:0.0:1767318690.699980:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582720 00000100:00100000:0.0:1767318690.700013:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582848 00000100:00100000:0.0:1767318690.700017:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582848 00000100:00100000:3.0:1767318690.700037:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 869 00000100:00100000:3.0:1767318690.700046:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915582720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318690.700067:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318690.700068:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 667 00000100:00100000:0.0:1767318690.700074:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915582848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318690.700139:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915582720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (248us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318690.700148:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 869 00000100:00080000:2.0:1767318690.700155:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318690.704484:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915582848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 4397us (4527us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318690.704499:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 667 00010000:00010000:1.0:1767318690.948815:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318691.268623:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318691.268657:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982625664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318691.268728:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318691.268817:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625664 00000100:00100000:1.0:1767318691.268824:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625664 00000100:00100000:1.0:1767318691.268876:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 466 00000100:00100000:1.0:1767318691.268883:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982625664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318691.268909:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318691.268945:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982625664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318691.268950:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 466 00000100:00080000:2.0:1767318691.269043:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318691.269088:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982625664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318691.269419:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982625792:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318691.269457:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318691.269512:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625792 00000100:00100000:3.0:1767318691.269518:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625792 00000100:00100000:3.0:1767318691.269569:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 467 00000100:00100000:3.0:1767318691.269576:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982625792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318691.269593:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767318691.269680:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318691.269907:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982625792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 331us (449us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318691.269912:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 467 00000100:00100000:2.0:1767318691.270011:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982625792:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318691.972975:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318692.996749:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318693.830353:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318693.830367:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318693.830443:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318693.830448:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318693.830452:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318693.830460:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3937 00000100:00080000:2.0:1767318693.830466:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318693.830475:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318693.830496:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982626048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318693.830529:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318693.830532:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318693.830534:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318693.830537:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3937 00000100:00080000:2.0:1767318693.830539:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318693.830543:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318693.830551:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626a00 x1853163982626304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318693.830558:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318693.830562:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318693.830563:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318693.830566:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3937 00000100:00080000:2.0:1767318693.830568:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318693.830571:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318693.830581:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627100 x1853163982626432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318693.830625:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318693.830629:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318693.830630:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318693.830633:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3937 00000100:00080000:2.0:1767318693.830636:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318693.830639:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318693.830649:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624e00 x1853163982626816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318693.830678:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982626560:0@lo:400:kworker.0 00000100:00100000:2.0:1767318693.830756:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318693.830796:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318693.830803:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982626688:0@lo:400:kworker.0 00000100:00100000:2.0:1767318693.830826:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318693.830859:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626560 00000100:00100000:2.0:1767318693.830868:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626560 00000100:00100000:2.0:1767318693.830930:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626688 00000100:00100000:2.0:1767318693.830934:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626688 00000100:00100000:2.0:1767318693.830959:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 437 00000100:00100000:3.0:1767318693.831103:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982626048:0@lo:38:kworker.0 00000100:00100000:3.0:1767318693.831138:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318693.831160:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982626304:0@lo:38:kworker.0 00000100:00100000:3.0:1767318693.831180:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318693.831199:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982626432:0@lo:38:kworker.0 00000100:00100000:3.0:1767318693.831221:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318693.831240:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982626816:0@lo:38:kworker.0 00000100:00100000:3.0:1767318693.831261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318693.831319:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626048 00000100:00100000:3.0:1767318693.831323:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626048 00000100:00100000:3.0:1767318693.831345:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626304 00000100:00100000:3.0:1767318693.831348:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626304 00000100:00100000:3.0:1767318693.831361:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626816 00000100:00100000:3.0:1767318693.831364:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626816 00000100:00100000:3.0:1767318693.831375:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 870 00000100:00100000:3.0:1767318693.831380:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318693.831399:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318693.831438:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626048:12345-0@lo:38:kworker.0 Request processed in 56us (299us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318693.831444:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 870 00000100:00100000:3.0:1767318693.831459:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 871 00000100:00100000:3.0:1767318693.831463:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318693.831473:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318693.831496:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626304:12345-0@lo:38:kworker.0 Request processed in 31us (315us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318693.831501:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 871 00000100:00100000:3.0:1767318693.831512:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 872 00000100:00100000:3.0:1767318693.831515:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318693.831522:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318693.831541:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982626816:12345-0@lo:38:kworker.0 Request processed in 25us (280us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318693.831547:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 872 00000100:00100000:3.0:1767318693.831571:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626432 00000100:00100000:3.0:1767318693.831575:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626432 00000100:00100000:3.0:1767318693.831610:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 243 00000100:00100000:3.0:1767318693.831614:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982626432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318693.831623:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318693.831650:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982626432:12345-0@lo:38:kworker.0 Request processed in 35us (428us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318693.831655:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 243 00000100:00080000:3.0:1767318693.831689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318693.831693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318693.831696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982626048:0@lo:38:kworker.0 00000100:00080000:3.0:1767318693.831704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318693.831707:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318693.831709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982626304:0@lo:38:kworker.0 00000100:00080000:3.0:1767318693.831716:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318693.831718:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318693.831721:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982626432:0@lo:38:kworker.0 00000100:00080000:3.0:1767318693.831728:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318693.831730:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318693.831733:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982626816:0@lo:38:kworker.0 00000100:00100000:3.0:1767318693.831779:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 438 00000100:00100000:3.0:1767318693.831786:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982626688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318693.831826:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982626688:12345-0@lo:400:kworker.0 Request processed in 41us (999us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318693.831833:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 438 00000100:00100000:3.0:1767318693.831864:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982626688:0@lo:400:kworker.0 00000100:00100000:0.0:1767318693.832108:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982625920:0@lo:400:kworker.0 00000100:00100000:0.0:1767318693.832139:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318693.832174:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318693.832180:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982626176:0@lo:400:kworker.0 00000100:00100000:0.0:1767318693.832200:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318693.832224:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982625920 00000100:00100000:0.0:1767318693.832227:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982625920 00000100:00100000:0.0:1767318693.832255:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626176 00000100:00100000:0.0:1767318693.832258:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626176 00000100:00100000:0.0:1767318693.832298:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 668 00000100:00100000:0.0:1767318693.832304:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982625920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318693.832338:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982625920:12345-0@lo:400:kworker.0 Request processed in 34us (198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318693.832344:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 668 00000100:00100000:0.0:1767318693.832356:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 669 00000100:00100000:0.0:1767318693.832362:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982626176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318693.832387:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982626176:12345-0@lo:400:kworker.0 Request processed in 25us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318693.832392:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 669 00010000:00080000:0.0:1767318693.832421:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624700 x1853163982626176/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318709 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318693.832430:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982626176:0@lo:400:kworker.0 00010000:00080000:0.0:1767318693.832439:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583480 x1853163982625920/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318709 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318693.832447:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982625920:0@lo:400:kworker.0 00000100:00100000:2.0:1767318693.832562:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982626560:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318693.832688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318693.832693:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318693.832696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318693.832742:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982626560:12345-0@lo:400:kworker.0 Request processed in 1774us (1997us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318693.832759:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 437 00000100:00100000:3.0:1767318693.836084:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982626560:0@lo:400:kworker.0 00010000:00010000:1.0:1767318694.023534:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318695.044444:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318695.820339:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318695.820426:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583104 00000100:00100000:3.0:1767318695.820436:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583104 00000100:00100000:3.0:1767318695.820511:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 670 00000100:00100000:3.0:1767318695.820519:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915583104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318695.820708:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915583104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 188us (380us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318695.820716:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 670 00000100:00100000:3.0:1767318695.820805:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318695.820876:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915582976 00000100:00100000:3.0:1767318695.820880:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915582976 00000100:00100000:3.0:1767318695.820898:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 873 00000100:00100000:3.0:1767318695.820903:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915582976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318695.820929:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318695.820960:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915582976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (155us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318695.820967:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 873 00000100:00080000:1.0:1767318695.821573:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318696.068572:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318696.390265:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318696.390307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982626944:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318696.390378:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318696.390724:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982626944 00000100:00100000:3.0:1767318696.390733:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982626944 00000100:00100000:3.0:1767318696.390804:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 468 00000100:00100000:3.0:1767318696.390814:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982626944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318696.390844:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318696.390888:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982626944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318696.390895:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 468 00000100:00080000:2.0:1767318696.390977:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318696.391024:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982626944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318696.396432:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318696.396463:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982627072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318696.396566:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318696.396788:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627072 00000100:00100000:3.0:1767318696.396801:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627072 00000100:00100000:3.0:1767318696.396894:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 469 00000100:00100000:3.0:1767318696.396904:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982627072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318696.396935:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318696.396983:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982627072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (417us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318696.396990:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 469 00000100:00080000:1.0:1767318696.398353:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318696.398416:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982627072:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318697.095725:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318698.116564:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318698.948421:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318698.948433:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318698.948510:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318698.948515:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318698.948519:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318698.948528:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3942 00000100:00080000:2.0:1767318698.948533:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318698.948541:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318698.948569:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163982627328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318698.948611:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767318698.948613:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318698.948618:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318698.948620:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318698.948625:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3942 00000100:00080000:2.0:1767318698.948628:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767318698.948630:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982627456:0@lo:400:kworker.0 00000100:00080000:2.0:1767318698.948635:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318698.948650:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163982627584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318698.948659:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318698.948663:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318698.948665:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318698.948669:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3942 00000100:00080000:2.0:1767318698.948671:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318698.948675:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318698.948688:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624380 x1853163982627712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318698.948711:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318698.948727:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318698.948731:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318698.948734:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318698.948737:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3942 00000100:00080000:2.0:1767318698.948740:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318698.948744:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318698.948755:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982627200:0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.948759:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624e00 x1853163982628096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318698.948792:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318698.948792:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982627968:0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.948838:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318698.948843:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627456 00000100:00100000:0.0:1767318698.948852:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627456 00000100:00100000:2.0:1767318698.948866:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318698.948873:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982627840:0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.948900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318698.948906:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627200 00000100:00100000:0.0:1767318698.948911:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627200 00000100:00100000:2.0:1767318698.948931:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627968 00000100:00100000:2.0:1767318698.948937:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627968 00000100:00100000:0.0:1767318698.948940:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00100000:0.0:1767318698.948952:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982627456:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.948971:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627840 00000100:00100000:2.0:1767318698.948976:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627840 00000100:00100000:2.0:1767318698.949000:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 439 00000100:00100000:0.0:1767318698.949008:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982627456:12345-0@lo:400:kworker.0 Request processed in 58us (298us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318698.949010:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982627968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318698.949024:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00100000:2.0:1767318698.949059:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982627968:12345-0@lo:400:kworker.0 Request processed in 50us (220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318698.949062:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00100000:2.0:1767318698.949074:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 439 00000100:00100000:0.0:1767318698.949076:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982627200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.949094:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 440 00000100:00100000:2.0:1767318698.949104:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982627840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318698.949110:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982627200:12345-0@lo:400:kworker.0 Request processed in 39us (321us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318698.949117:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00100000:2.0:1767318698.949155:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982627840:12345-0@lo:400:kworker.0 Request processed in 53us (254us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318698.949163:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 440 00010000:00080000:0.0:1767318698.949169:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624a80 x1853163982627456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318714 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318698.949185:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982627456:0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.949202:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982627840:0@lo:400:kworker.0 00010000:00080000:0.0:1767318698.949203:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624000 x1853163982627200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318714 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318698.949218:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982627200:0@lo:400:kworker.0 00000100:00100000:2.0:1767318698.949222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982627968:0@lo:400:kworker.0 00000100:00080000:1.0:1767318698.949538:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318698.949544:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318698.949548:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318698.949556:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982627328:0@lo:38:kworker.0 00000100:00100000:3.0:1767318698.949592:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318698.949623:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982627584:0@lo:38:kworker.0 00000100:00100000:3.0:1767318698.949643:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318698.949671:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982627712:0@lo:38:kworker.0 00000100:00100000:3.0:1767318698.949697:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318698.949740:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982628096:0@lo:38:kworker.0 00000100:00100000:3.0:1767318698.949763:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318698.949790:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627328 00000100:00100000:3.0:1767318698.949795:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627328 00000100:00100000:3.0:1767318698.949817:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627584 00000100:00100000:3.0:1767318698.949820:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627584 00000100:00100000:2.0:1767318698.950306:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982627712 00000100:00100000:1.0:1767318698.950313:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628096 00000100:00100000:1.0:1767318698.950319:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628096 00000100:00100000:2.0:1767318698.950329:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982627712 00000100:00100000:1.0:1767318698.950350:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767318698.950358:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982627328:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318698.950363:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 244 00000100:00100000:2.0:1767318698.950371:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604355050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982627712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318698.950508:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318698.950566:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604355050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982627712:12345-0@lo:38:kworker.0 Request processed in 193us (866us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318698.950575:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 244 00010000:00020000:1.0:1767318698.950674:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318698.950711:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982627328:12345-0@lo:38:kworker.0 Request processed in 352us (1117us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318698.950720:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767318698.950739:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 875 00000100:00100000:1.0:1767318698.950744:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982628096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318698.950755:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318698.950785:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982628096:12345-0@lo:38:kworker.0 Request processed in 40us (1021us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318698.950791:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 875 00000100:00080000:2.0:1767318698.950810:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318698.950818:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318698.950822:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982627328:0@lo:38:kworker.0 00000100:00080000:2.0:1767318698.950833:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318698.950836:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318698.950839:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982627712:0@lo:38:kworker.0 00000100:00080000:2.0:1767318698.950849:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318698.950853:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318698.950857:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982628096:0@lo:38:kworker.0 00000100:00100000:3.0:1767318698.951318:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 876 00000100:00100000:3.0:1767318698.951325:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982627584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318698.951336:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318698.951374:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982627584:12345-0@lo:38:kworker.0 Request processed in 48us (1730us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318698.951381:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 876 00000100:00080000:2.0:1767318698.951667:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318698.951672:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318698.951676:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982627584:0@lo:38:kworker.0 00010000:00010000:1.0:1767318699.141138:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318700.165848:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318700.942190:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318700.942339:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318700.942380:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583360 00000100:00100000:0.0:1767318700.942392:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583360 00000100:00080000:2.0:1767318700.942480:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318700.942482:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583232 00000100:00100000:0.0:1767318700.942488:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583232 00000100:00100000:0.0:1767318700.942520:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 673 00000100:00100000:0.0:1767318700.942531:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915583360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318700.942557:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 877 00000100:00100000:2.0:1767318700.942566:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915583232:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318700.942589:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318700.942618:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915583360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318700.942631:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 673 00000100:00100000:2.0:1767318700.942631:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915583232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (291us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318700.942640:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 877 00010000:00010000:1.0:1767318701.188908:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318701.510398:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982628224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318701.510434:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982628352:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318701.510498:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318701.511018:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318701.511116:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628224 00000100:00100000:0.0:1767318701.511127:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628224 00000100:00100000:0.0:1767318701.511208:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628352 00000100:00100000:0.0:1767318701.511211:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628352 00000100:00100000:0.0:1767318701.511230:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 470 00000100:00100000:0.0:1767318701.511240:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982628224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318701.511302:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767318701.511306:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318701.511312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318701.511369:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982628224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 130us (870us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318701.511377:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 471 00000100:00100000:2.0:1767318701.511463:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982628224:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318701.511622:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982628352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318701.511641:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318701.511747:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982628352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 361us (729us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318701.515913:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 470 00000100:00100000:1.0:1767318701.519064:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982628352:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318701.521018:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 471 00010000:00010000:1.0:1767318702.218068:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318703.236587:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318704.069731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.069742:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318704.069815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.069819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.069823:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.069831:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.069836:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.069843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318704.069866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163982628608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318704.069885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.069888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.069890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.069894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.069896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.069899:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318704.069907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163982628864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318704.069915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.069918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.069920:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.069923:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.069926:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.069929:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318704.069937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4e00 x1853163982628992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318704.069970:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982628736:0@lo:400:kworker.0 00000100:00080000:3.0:1767318704.069975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318704.069980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318704.069982:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318704.069986:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3947 00000100:00080000:3.0:1767318704.069988:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318704.069992:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318704.070002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163982629376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318704.070031:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982629120:0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070057:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318704.070070:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318704.070133:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318704.070140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982628480:0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070161:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318704.070198:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628736 00000100:00100000:0.0:1767318704.070205:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628736 00000100:00100000:0.0:1767318704.070253:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628480 00000100:00100000:0.0:1767318704.070256:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628480 00000100:00100000:0.0:1767318704.070301:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 674 00000100:00100000:0.0:1767318704.070310:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982628736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070357:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982628736:12345-0@lo:400:kworker.0 Request processed in 49us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318704.070365:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 674 00000100:00100000:0.0:1767318704.070380:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 675 00000100:00100000:0.0:1767318704.070386:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982628480:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070422:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982628480:12345-0@lo:400:kworker.0 Request processed in 35us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318704.070428:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 675 00000100:00080000:0.0:1767318704.070444:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318704.070476:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee4380 x1853163982628480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318720 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318704.070494:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982628480:0@lo:400:kworker.0 00010000:00080000:0.0:1767318704.070503:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6300 x1853163982628736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318720 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318704.070511:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982628736:0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070547:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982629248:0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070570:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318704.070599:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629120 00000100:00100000:0.0:1767318704.070602:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629120 00000100:00100000:0.0:1767318704.070627:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629248 00000100:00100000:0.0:1767318704.070630:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629248 00000100:00100000:0.0:1767318704.070647:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 441 00000100:00100000:0.0:1767318704.070652:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982629120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070678:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982629120:12345-0@lo:400:kworker.0 Request processed in 26us (610us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318704.070684:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 441 00000100:00100000:0.0:1767318704.070693:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 442 00000100:00100000:0.0:1767318704.070699:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982629248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318704.070722:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982629248:12345-0@lo:400:kworker.0 Request processed in 24us (152us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318704.070728:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 442 00000100:00080000:0.0:1767318704.070739:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318704.070742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318704.070754:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982629248:0@lo:400:kworker.0 00000100:00100000:2.0:1767318704.071304:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982628608:0@lo:38:kworker.0 00000100:00100000:2.0:1767318704.071344:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318704.071383:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982628864:0@lo:38:kworker.0 00000100:00100000:2.0:1767318704.071414:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318704.071448:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982628992:0@lo:38:kworker.0 00000100:00100000:2.0:1767318704.071518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318704.071541:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982629376:0@lo:38:kworker.0 00000100:00100000:2.0:1767318704.071560:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318704.071589:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628608 00000100:00100000:2.0:1767318704.071595:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628608 00000100:00100000:2.0:1767318704.071615:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628864 00000100:00100000:2.0:1767318704.071618:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628864 00000100:00100000:2.0:1767318704.071627:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629376 00000100:00100000:2.0:1767318704.071629:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629376 00000100:00100000:2.0:1767318704.071638:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 878 00000100:00100000:2.0:1767318704.071644:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982628608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318704.071664:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318704.071698:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982628608:12345-0@lo:38:kworker.0 Request processed in 53us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318704.071705:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 878 00000100:00100000:2.0:1767318704.071717:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767318704.071721:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982628864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318704.071729:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318704.071746:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982628864:12345-0@lo:38:kworker.0 Request processed in 25us (332us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318704.071752:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767318704.071761:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 880 00000100:00100000:2.0:1767318704.071764:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982629376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318704.071774:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318704.071808:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982629376:12345-0@lo:38:kworker.0 Request processed in 42us (247us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318704.071814:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 880 00000100:00080000:2.0:1767318704.071845:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318704.071850:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318704.071853:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982628608:0@lo:38:kworker.0 00000100:00080000:2.0:1767318704.071860:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318704.071863:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318704.071866:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982628864:0@lo:38:kworker.0 00000100:00080000:2.0:1767318704.071872:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318704.071875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318704.071877:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982629376:0@lo:38:kworker.0 00000100:00100000:1.0:1767318704.074389:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982628992 00000100:00100000:1.0:1767318704.074419:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982628992 00000100:00100000:1.0:1767318704.074477:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 245 00000100:00100000:1.0:1767318704.074485:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982628992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318704.074508:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318704.074596:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982628992:12345-0@lo:38:kworker.0 Request processed in 111us (3077us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318704.074607:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 245 00000100:00080000:0.0:1767318704.075001:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318704.075008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318704.075012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982628992:0@lo:38:kworker.0 00000100:00100000:3.0:1767318704.076318:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982629120:0@lo:400:kworker.0 00010000:00010000:1.0:1767318704.261929:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318705.284530:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318706.065788:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318706.065890:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318706.065980:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583616 00000100:00100000:3.0:1767318706.065992:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583616 00000100:00100000:3.0:1767318706.066063:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583488 00000100:00100000:3.0:1767318706.066066:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583488 00000100:00100000:3.0:1767318706.066090:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 676 00000100:00100000:3.0:1767318706.066101:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915583616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318706.066245:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915583616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 145us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318706.066253:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 676 00000100:00080000:0.0:1767318706.066443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318706.066784:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 881 00000100:00100000:3.0:1767318706.066791:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915583488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318706.066811:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318706.066898:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915583488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 105us (1006us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318706.066906:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 881 00010000:00010000:1.0:1767318706.312258:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318706.628660:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318706.628690:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982629504:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318706.628770:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318706.630240:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629504 00000100:00100000:3.0:1767318706.630252:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629504 00000100:00100000:3.0:1767318706.631264:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 472 00000100:00080000:0.0:1767318706.631860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318706.631883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982629632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318706.631951:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318706.632006:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629632 00000100:00100000:0.0:1767318706.632013:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629632 00000100:00100000:3.0:1767318706.632183:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982629504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318706.632213:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318706.632264:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982629504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (3492us total) trans 0 rc 0/0 00000100:00080000:1.0:1767318706.632657:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318706.632703:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982629504:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318706.632979:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 473 00000100:00100000:0.0:1767318706.632992:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982629632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318706.633011:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318706.633150:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 472 00000100:00100000:0.0:1767318706.633246:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982629632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 258us (1294us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318706.633254:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 473 00000100:00100000:2.0:1767318706.642611:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982629632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318707.336833:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318708.356680:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318709.188458:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318709.188477:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318709.188546:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318709.188550:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318709.188554:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318709.188562:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3952 00000100:00080000:2.0:1767318709.188567:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318709.188573:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318709.188596:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624700 x1853163982629888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318709.188617:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318709.188621:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318709.188622:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318709.188626:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3952 00000100:00080000:2.0:1767318709.188628:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318709.188632:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318709.188641:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624000 x1853163982630144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318709.188650:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318709.188653:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318709.188655:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318709.188658:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3952 00000100:00080000:2.0:1767318709.188660:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318709.188663:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318709.188673:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624a80 x1853163982630272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318709.188716:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318709.188719:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318709.188721:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318709.188724:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3952 00000100:00080000:2.0:1767318709.188726:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318709.188729:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318709.188739:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966a00 x1853163982630656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318709.188768:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982630400:0@lo:400:kworker.0 00000100:00100000:2.0:1767318709.188832:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318709.188891:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318709.188897:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982630528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.189195:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982629760:0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.189227:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.189254:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318709.189260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982630016:0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.189300:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318709.189347:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629760 00000100:00100000:3.0:1767318709.189354:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629760 00000100:00100000:3.0:1767318709.189402:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630016 00000100:00100000:3.0:1767318709.189405:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630016 00000100:00100000:3.0:1767318709.189431:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 677 00000100:00100000:3.0:1767318709.189439:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982629760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.189486:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982629760:12345-0@lo:400:kworker.0 Request processed in 48us (258us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318709.189493:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 677 00000100:00100000:3.0:1767318709.189522:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 678 00000100:00100000:3.0:1767318709.189529:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982630016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318709.189555:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982630016:12345-0@lo:400:kworker.0 Request processed in 27us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318709.189561:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 678 00010000:00080000:3.0:1767318709.189601:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625180 x1853163982630016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318725 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318709.189615:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982630016:0@lo:400:kworker.0 00010000:00080000:3.0:1767318709.189626:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625c00 x1853163982629760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318725 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318709.189633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982629760:0@lo:400:kworker.0 00000100:00100000:2.0:1767318709.190034:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318709.190145:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318709.190246:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630400 00000100:00100000:0.0:1767318709.190253:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630400 00000100:00100000:0.0:1767318709.190318:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630528 00000100:00100000:0.0:1767318709.190322:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630528 00000100:00100000:0.0:1767318709.190345:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 443 00000100:00100000:0.0:1767318709.190352:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982630400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318709.190403:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982630400:12345-0@lo:400:kworker.0 Request processed in 51us (1571us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318709.190413:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 443 00000100:00100000:0.0:1767318709.190426:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 444 00000100:00100000:0.0:1767318709.190432:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982630528:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318709.190457:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982630528:12345-0@lo:400:kworker.0 Request processed in 26us (1537us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318709.190463:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 444 00000100:00080000:1.0:1767318709.190624:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318709.190628:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318709.190660:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982629888:0@lo:38:kworker.0 00000100:00100000:1.0:1767318709.190705:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318709.190733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982630144:0@lo:38:kworker.0 00000100:00100000:1.0:1767318709.190754:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318709.190770:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982630272:0@lo:38:kworker.0 00000100:00100000:1.0:1767318709.190792:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318709.190815:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982630656:0@lo:38:kworker.0 00000100:00100000:1.0:1767318709.190837:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318709.190865:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630272 00000100:00100000:1.0:1767318709.190870:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630272 00000100:00100000:1.0:1767318709.190890:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 246 00000100:00100000:1.0:1767318709.190895:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982630272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318709.190911:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318709.191124:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982630272:12345-0@lo:38:kworker.0 Request processed in 226us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318709.191134:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 246 00000100:00100000:1.0:1767318709.191158:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982629888 00000100:00100000:1.0:1767318709.191161:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982629888 00000100:00100000:1.0:1767318709.191178:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630144 00000100:00100000:1.0:1767318709.191180:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630144 00000100:00100000:1.0:1767318709.191191:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630656 00000100:00100000:1.0:1767318709.191193:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630656 00000100:00100000:1.0:1767318709.191203:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 882 00000100:00100000:1.0:1767318709.191208:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982629888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318709.191218:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318709.191243:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982629888:12345-0@lo:38:kworker.0 Request processed in 33us (536us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318709.191249:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 882 00000100:00100000:1.0:1767318709.191260:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 883 00000100:00100000:1.0:1767318709.191263:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982630144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318709.191297:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318709.191333:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982630144:12345-0@lo:38:kworker.0 Request processed in 68us (578us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318709.191338:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 883 00000100:00100000:1.0:1767318709.191350:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 884 00000100:00100000:1.0:1767318709.191354:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982630656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318709.191363:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318709.191384:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982630656:12345-0@lo:38:kworker.0 Request processed in 29us (546us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318709.191390:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 884 00000100:00080000:1.0:1767318709.191432:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318709.191437:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318709.191440:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982629888:0@lo:38:kworker.0 00000100:00080000:1.0:1767318709.191449:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318709.191452:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318709.191455:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982630144:0@lo:38:kworker.0 00000100:00080000:1.0:1767318709.191463:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318709.191465:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318709.191468:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982630272:0@lo:38:kworker.0 00000100:00100000:0.0:1767318709.191775:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982630528:0@lo:400:kworker.0 00000100:00100000:0.0:1767318709.191797:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982630400:0@lo:400:kworker.0 00000100:00080000:1.0:1767318709.191876:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318709.191880:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318709.191883:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982630656:0@lo:38:kworker.0 00010000:00010000:1.0:1767318709.380708:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767318710.157189:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915583744 offset 0 length 520: 4 00000100:00100000:0.0:1767318710.158371:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318710.158411:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318710.160871:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915583872 00000100:00100000:2.0:1767318710.160892:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915583872 00000100:00100000:2.0:1767318710.161445:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584000 00000100:00100000:2.0:1767318710.161450:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584000 00000100:00100000:2.0:1767318710.161514:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 679 00000100:00100000:2.0:1767318710.161525:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915584000:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318710.161614:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915584000:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (3202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318710.161622:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 679 00000100:00080000:0.0:1767318710.161732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318710.161762:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 885 00000100:00100000:0.0:1767318710.161768:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915583872:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318710.161791:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318710.161821:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915583872:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (3465us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318710.161827:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 885 00010000:00010000:1.0:1767318710.404674:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318711.428576:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318711.748622:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982630784:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318711.748695:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318711.748893:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630784 00000100:00100000:1.0:1767318711.748902:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630784 00000100:00080000:2.0:1767318711.748957:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318711.748971:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 474 00000100:00100000:1.0:1767318711.748981:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982630784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318711.749011:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318711.749068:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982630784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (372us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318711.749075:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 474 00000100:00100000:2.0:1767318711.749323:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982630784:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318711.749675:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318711.749686:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982630912:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318711.749720:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318711.749873:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982630912 00000100:00100000:0.0:1767318711.749879:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982630912 00000100:00100000:0.0:1767318711.749934:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 475 00000100:00100000:0.0:1767318711.749941:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982630912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318711.749960:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767318711.749960:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318711.750041:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982630912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318711.750051:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 475 00000100:00100000:3.0:1767318711.759547:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982630912:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318712.455164:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318713.284411:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318688/real 1767318688] req@ffff93460543d180 x1853163982624512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318713 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318713.284452:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460543d180 x1853163982624512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318713 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318713.284467:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.284494:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318713.284498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982624512:0@lo:250:kworker.0 00000100:00080000:3.0:1767318713.284585:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318713.284591:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767318713.284594:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318713.284599:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3937 00000100:00080000:3.0:1767318713.284603:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318713.284610:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318713.284643:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624000 x1853163982631040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318713.284710:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318713.284714:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318713.284716:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318713.284720:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3957 00000100:00080000:3.0:1767318713.284722:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318713.284726:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318713.284746:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625c00 x1853163982631296/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318713.284762:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318713.284765:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318713.284767:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318713.284770:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3957 00000100:00080000:3.0:1767318713.284772:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318713.284775:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318713.284786:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163982631552/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318713.284794:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318713.284797:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318713.284799:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318713.284802:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3957 00000100:00080000:3.0:1767318713.284804:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318713.284807:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318713.284817:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163982631680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318713.284866:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318713.284870:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318713.284872:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318713.284875:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3957 00000100:00080000:3.0:1767318713.284877:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318713.284881:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318713.284893:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36680 x1853163982632064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318713.284919:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982631808:0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.284985:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318713.285044:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318713.285050:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982631936:0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.285070:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318713.285097:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631808 00000100:00100000:3.0:1767318713.285105:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631808 00000100:00100000:3.0:1767318713.285150:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631936 00000100:00100000:3.0:1767318713.285153:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631936 00000100:00100000:3.0:1767318713.285173:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 445 00000100:00100000:3.0:1767318713.285182:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982631808:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.285265:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982631808:12345-0@lo:400:kworker.0 Request processed in 84us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318713.285301:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 445 00000100:00100000:3.0:1767318713.285318:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 446 00000100:00100000:3.0:1767318713.285324:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982631936:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.285354:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982631936:12345-0@lo:400:kworker.0 Request processed in 30us (282us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318713.285359:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 446 00000100:00100000:3.0:1767318713.285392:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982631936:0@lo:400:kworker.0 00000100:00100000:3.0:1767318713.285401:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982631808:0@lo:400:kworker.0 00000100:00100000:0.0:1767318713.285478:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982631168:0@lo:400:kworker.0 00000100:00100000:0.0:1767318713.285510:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318713.285546:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318713.285551:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982631424:0@lo:400:kworker.0 00000100:00100000:0.0:1767318713.285573:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.285822:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982631040:0@lo:250:kworker.0 00000400:00000100:1.0:1767318713.285861:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982631040 offset 0 length 520: 4 00000100:00100000:1.0:1767318713.285882:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982631296:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.285902:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.285929:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982631552:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.285950:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.285963:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982631680:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.285983:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.286001:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982632064:0@lo:38:kworker.0 00000100:00100000:1.0:1767318713.286019:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318713.286077:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631680 00000100:00100000:1.0:1767318713.286082:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631680 00000100:00100000:1.0:1767318713.286108:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 247 00000100:00100000:1.0:1767318713.286114:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982631680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.286135:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.286170:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982631680:12345-0@lo:38:kworker.0 Request processed in 55us (186us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.286176:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 247 00000100:00100000:1.0:1767318713.286197:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631168 00000100:00100000:1.0:1767318713.286201:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631168 00000100:00100000:1.0:1767318713.286222:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631424 00000100:00100000:1.0:1767318713.286225:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631424 00000100:00100000:1.0:1767318713.286250:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631296 00000100:00100000:1.0:1767318713.286253:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631296 00000100:00100000:1.0:1767318713.286260:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982631552 00000100:00100000:1.0:1767318713.286263:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982631552 00000100:00100000:1.0:1767318713.286292:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632064 00000100:00100000:1.0:1767318713.286295:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632064 00000100:00100000:1.0:1767318713.286305:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 680 00000100:00100000:1.0:1767318713.286312:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982631168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.286348:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982631168:12345-0@lo:400:kworker.0 Request processed in 36us (837us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318713.286354:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 680 00000100:00100000:1.0:1767318713.286378:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 681 00000100:00100000:1.0:1767318713.286384:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982631424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318713.286419:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982631424:12345-0@lo:400:kworker.0 Request processed in 27us (836us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318713.286424:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 681 00000100:00100000:1.0:1767318713.286435:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 886 00000100:00100000:1.0:1767318713.286445:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982631296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.286456:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.286479:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982631296:12345-0@lo:38:kworker.0 Request processed in 39us (576us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.286486:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 886 00000100:00100000:1.0:1767318713.286496:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 887 00000100:00100000:1.0:1767318713.286500:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982631552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.286510:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.286529:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982631552:12345-0@lo:38:kworker.0 Request processed in 27us (578us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.286534:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 887 00000100:00100000:1.0:1767318713.286544:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 888 00000100:00100000:1.0:1767318713.286548:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982632064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318713.286556:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318713.286626:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982632064:12345-0@lo:38:kworker.0 Request processed in 76us (605us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318713.286632:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 888 00000100:00080000:1.0:1767318713.286669:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.286674:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.286677:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982631296:0@lo:38:kworker.0 00010000:00080000:0.0:1767318713.286902:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625180 x1853163982631424/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318729 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318713.286914:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982631424:0@lo:400:kworker.0 00010000:00080000:0.0:1767318713.286924:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624700 x1853163982631168/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318729 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318713.286932:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982631168:0@lo:400:kworker.0 00000100:00080000:1.0:1767318713.287194:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.287198:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.287201:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982631552:0@lo:38:kworker.0 00000100:00080000:1.0:1767318713.287209:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.287212:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.287214:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982631680:0@lo:38:kworker.0 00000100:00080000:1.0:1767318713.287222:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318713.287225:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318713.287236:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982632064:0@lo:38:kworker.0 00000100:00080000:2.0:1767318713.287685:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318713.287693:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318713.287696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318713.477070:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318714.500876:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318715.282725:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318715.282775:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318715.282887:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584128 00000100:00100000:2.0:1767318715.282897:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584128 00000100:00100000:2.0:1767318715.282951:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584256 00000100:00100000:2.0:1767318715.282955:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584256 00000100:00100000:2.0:1767318715.283005:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 682 00000100:00100000:2.0:1767318715.283017:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915584256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318715.283106:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915584256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (331us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318715.283114:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 682 00000100:00100000:2.0:1767318715.283131:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 889 00000100:00100000:2.0:1767318715.283135:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318715.283155:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318715.283180:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (467us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318715.283187:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 889 00000100:00080000:2.0:1767318715.283214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318715.526552:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318716.550127:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318716.868573:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982632320:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318716.868650:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318716.868691:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982632192:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318716.868713:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318716.868808:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632320 00000100:00100000:0.0:1767318716.868816:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632320 00000100:00100000:0.0:1767318716.868894:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632192 00000100:00100000:0.0:1767318716.868898:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632192 00000100:00100000:0.0:1767318716.868919:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 476 00000100:00100000:0.0:1767318716.868928:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982632320:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318716.868957:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318716.869011:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982632320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318716.869019:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 476 00000100:00100000:0.0:1767318716.869036:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 477 00000100:00100000:0.0:1767318716.869043:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982632192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318716.869061:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318716.869086:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982632192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318716.869092:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 477 00000100:00100000:0.0:1767318716.869131:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982632320:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318716.869146:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982632192:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318716.869290:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318716.869295:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318717.572864:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318718.404795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318718.404805:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318718.404860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318718.404864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318718.404868:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318718.404876:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3961 00000100:00080000:1.0:1767318718.404880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318718.404888:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318718.404903:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318718.404913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1180 x1853163982632576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318718.404922:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982632448:0@lo:400:kworker.0 00000100:00080000:1.0:1767318718.404957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318718.404961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318718.404963:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318718.404968:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3961 00000100:00080000:1.0:1767318718.404971:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318718.404975:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318718.404987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0000 x1853163982632832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318718.404997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318718.405001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318718.405003:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318718.405007:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3961 00000100:00080000:1.0:1767318718.405009:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318718.405013:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318718.405024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2680 x1853163982632960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318718.405045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318718.405049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318718.405051:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318718.405055:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3961 00000100:00080000:1.0:1767318718.405058:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318718.405061:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318718.405076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163982633344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318718.405183:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982632576:0@lo:38:kworker.0 00000100:00100000:3.0:1767318718.405258:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982632704:0@lo:400:kworker.0 00000100:00100000:2.0:1767318718.407513:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318718.409131:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318718.409173:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982633216:0@lo:400:kworker.0 00000100:00100000:2.0:1767318718.409175:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982632832:0@lo:38:kworker.0 00000100:00100000:2.0:1767318718.409239:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318718.409244:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318718.409290:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982632960:0@lo:38:kworker.0 00000100:00100000:0.0:1767318718.409316:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633216 00000100:00100000:0.0:1767318718.409324:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633216 00000100:00100000:0.0:1767318718.409376:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 447 00000100:00100000:0.0:1767318718.409384:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982633216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.409438:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982633216:12345-0@lo:400:kworker.0 Request processed in 55us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318718.409445:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 447 00000100:00100000:0.0:1767318718.409468:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632576 00000100:00100000:0.0:1767318718.409471:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632576 00000100:00100000:0.0:1767318718.409500:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632832 00000100:00100000:0.0:1767318718.409503:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632832 00000100:00100000:0.0:1767318718.409517:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 890 00000100:00100000:0.0:1767318718.409521:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982632576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318718.409550:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318718.409579:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982632576:12345-0@lo:38:kworker.0 Request processed in 56us (2074us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318718.409584:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 890 00000100:00100000:0.0:1767318718.409594:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 891 00000100:00100000:0.0:1767318718.409597:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982632832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318718.409605:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318718.409625:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982632832:12345-0@lo:38:kworker.0 Request processed in 27us (388us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318718.409631:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 891 00000100:00100000:0.0:1767318718.409669:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982633216:0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.409694:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318718.409717:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632448 00000100:00100000:0.0:1767318718.409720:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632448 00000100:00100000:0.0:1767318718.409741:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00100000:0.0:1767318718.409747:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982632448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.409777:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982632448:12345-0@lo:400:kworker.0 Request processed in 31us (83us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318718.409783:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00010000:00080000:0.0:1767318718.409805:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1f80 x1853163982632448/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318734 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318718.409817:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982632448:0@lo:400:kworker.0 00000100:00080000:0.0:1767318718.409842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318718.409846:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318718.409867:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982633088:0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.409890:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318718.409915:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633088 00000100:00100000:0.0:1767318718.409918:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633088 00000100:00100000:0.0:1767318718.409942:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 448 00000100:00100000:0.0:1767318718.409948:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982633088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.409980:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982633088:12345-0@lo:400:kworker.0 Request processed in 32us (88us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318718.409985:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 448 00000100:00080000:0.0:1767318718.409997:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318718.410016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982633088:0@lo:400:kworker.0 00000100:00100000:2.0:1767318718.411082:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318718.411119:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982633344:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.411159:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632960 00000100:00100000:1.0:1767318718.411166:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632960 00000100:00100000:2.0:1767318718.411167:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318718.411203:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 248 00000100:00100000:1.0:1767318718.411209:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982632960:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767318718.411215:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318718.411221:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:1.0:1767318718.411226:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318718.411228:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982632576:0@lo:38:kworker.0 00000100:00080000:2.0:1767318718.411240:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318718.411244:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318718.411248:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982632832:0@lo:38:kworker.0 00000100:00080000:2.0:1767318718.411291:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767318718.411297:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982632960:12345-0@lo:38:kworker.0 Request processed in 60us (191us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767318718.411299:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318718.411306:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982632960:0@lo:38:kworker.0 00000100:00100000:1.0:1767318718.411311:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 248 00000100:00100000:0.0:1767318718.411720:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633344 00000100:00100000:0.0:1767318718.411724:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633344 00000100:00100000:0.0:1767318718.411745:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 892 00000100:00100000:0.0:1767318718.411751:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982633344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318718.411761:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318718.411792:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982633344:12345-0@lo:38:kworker.0 Request processed in 40us (625us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318718.411798:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 892 00000100:00080000:2.0:1767318718.412453:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318718.412459:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318718.412462:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982633344:0@lo:38:kworker.0 00000100:00100000:3.0:1767318718.413046:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318718.413100:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982632704 00000100:00100000:0.0:1767318718.413106:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982632704 00000100:00080000:2.0:1767318718.413293:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318718.413376:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 684 00000100:00100000:0.0:1767318718.413388:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982632704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318718.413452:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982632704:12345-0@lo:400:kworker.0 Request processed in 67us (407us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318718.413460:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 684 00010000:00080000:1.0:1767318718.413529:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0380 x1853163982632704/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318734 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318718.413553:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982632704:0@lo:400:kworker.0 00010000:00010000:1.0:1767318718.596945:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318719.372399:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318719.372511:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584384 00000100:00100000:0.0:1767318719.372522:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584384 00000100:00100000:0.0:1767318719.372597:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 893 00000100:00100000:0.0:1767318719.372609:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915584384:12345-192.168.204.7@tcp:17:kworker.0 00000100:00100000:0.0:1767318719.372700:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915584384:12345-192.168.204.7@tcp:17:kworker.0 Request processed in 94us (314us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318719.372706:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 893 00000100:00080000:2.0:1767318719.372935:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318719.620445:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318720.397252:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318720.397347:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318720.397572:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584512 00000100:00100000:1.0:1767318720.397582:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584512 00000100:00100000:1.0:1767318720.397628:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584640 00000100:00100000:1.0:1767318720.397632:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584640 00000100:00100000:1.0:1767318720.397672:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 685 00000100:00100000:1.0:1767318720.397683:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915584640:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318720.397774:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915584640:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (427us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318720.397782:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 685 00000100:00100000:1.0:1767318720.397802:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 894 00000100:00100000:1.0:1767318720.397806:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584512:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318720.397829:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318720.397860:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584512:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (620us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318720.397866:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 894 00010000:00010000:1.0:1767318720.644969:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318721.668640:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318721.988555:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982633472:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318721.988632:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318721.989516:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982633600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318721.989556:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318721.989671:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633472 00000100:00100000:3.0:1767318721.989679:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633472 00000100:00100000:3.0:1767318721.989749:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633600 00000100:00100000:3.0:1767318721.989752:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633600 00000100:00100000:3.0:1767318721.989773:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 478 00000100:00100000:3.0:1767318721.989783:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982633472:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318721.989822:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318721.989879:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982633472:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (1246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318721.989886:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 478 00000100:00100000:3.0:1767318721.989902:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 479 00000100:00100000:3.0:1767318721.989908:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982633600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318721.989923:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318721.989951:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982633600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (395us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318721.989957:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 479 00000100:00100000:3.0:1767318721.989995:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982633472:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318721.990317:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318721.990322:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318721.990348:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982633600:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318722.692600:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318723.524394:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318723.524405:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318723.524466:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318723.524471:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318723.524476:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318723.524485:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3966 00000100:00080000:2.0:1767318723.524500:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318723.524509:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318723.524535:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54e00 x1853163982633856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318723.524580:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318723.524584:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318723.524586:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318723.524590:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3966 00000100:00080000:2.0:1767318723.524593:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318723.524597:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318723.524609:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982634112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318723.524617:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318723.524620:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318723.524622:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318723.524625:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3966 00000100:00080000:2.0:1767318723.524627:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318723.524630:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318723.524642:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54380 x1853163982634240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318723.524662:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982633856:0@lo:38:kworker.0 00000100:00080000:2.0:1767318723.524666:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318723.524671:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318723.524673:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318723.524676:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3966 00000100:00080000:2.0:1767318723.524679:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318723.524682:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318723.524694:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55500 x1853163982634624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318723.524720:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982633984:0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.524730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318723.524811:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982634496:0@lo:400:kworker.0 00000100:00100000:0.0:1767318723.524863:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982633728:0@lo:400:kworker.0 00000100:00100000:2.0:1767318723.528539:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318723.528607:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318723.528624:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982634368:0@lo:400:kworker.0 00000100:00100000:2.0:1767318723.528681:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318723.528734:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634368 00000100:00100000:2.0:1767318723.528742:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634368 00000100:00100000:2.0:1767318723.528816:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 449 00000100:00100000:2.0:1767318723.528827:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982634368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318723.528884:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982634368:12345-0@lo:400:kworker.0 Request processed in 58us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318723.528893:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 449 00000100:00080000:2.0:1767318723.528916:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318723.528947:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982634368:0@lo:400:kworker.0 00000100:00100000:2.0:1767318723.528991:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633856 00000100:00100000:2.0:1767318723.528995:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633856 00000100:00100000:2.0:1767318723.529013:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633984 00000100:00100000:2.0:1767318723.529015:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633984 00000100:00100000:2.0:1767318723.529054:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00100000:2.0:1767318723.529060:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982633984:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318723.529099:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982633984:12345-0@lo:400:kworker.0 Request processed in 39us (572us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318723.529105:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00100000:2.0:1767318723.529120:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 895 00000100:00100000:2.0:1767318723.529124:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982633856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318723.529145:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318723.529177:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982633856:12345-0@lo:38:kworker.0 Request processed in 52us (4448us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318723.529183:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 895 00000100:00080000:2.0:1767318723.529218:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318723.529224:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318723.529231:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318723.529235:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982633856:0@lo:38:kworker.0 00010000:00080000:2.0:1767318723.529243:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55c00 x1853163982633984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318739 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318723.529245:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982634112:0@lo:38:kworker.0 00000100:00100000:2.0:1767318723.529261:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982633984:0@lo:400:kworker.0 00000100:00100000:1.0:1767318723.529291:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318723.529313:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982634240:0@lo:38:kworker.0 00000100:00100000:2.0:1767318723.529316:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634112 00000100:00100000:2.0:1767318723.529322:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634112 00000100:00100000:1.0:1767318723.529343:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318723.529352:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896 00000100:00100000:2.0:1767318723.529359:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982634112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318723.529370:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318723.529384:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982634624:0@lo:38:kworker.0 00000100:00100000:2.0:1767318723.529410:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982634112:12345-0@lo:38:kworker.0 Request processed in 48us (117us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318723.529418:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318723.529422:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896 00000100:00100000:2.0:1767318723.529438:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634624 00000100:00100000:2.0:1767318723.529442:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634624 00000100:00100000:2.0:1767318723.529460:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 897 00000100:00100000:2.0:1767318723.529464:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982634624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318723.529475:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318723.529668:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318723.529734:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634240 00000100:00100000:0.0:1767318723.529739:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634240 00000100:00100000:0.0:1767318723.529763:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 249 00000100:00100000:0.0:1767318723.529769:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982634240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318723.529781:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318723.529812:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982634240:12345-0@lo:38:kworker.0 Request processed in 42us (470us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318723.529819:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 249 00000100:00100000:0.0:1767318723.529839:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982633728 00000100:00100000:0.0:1767318723.529843:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982633728 00000100:00100000:0.0:1767318723.529873:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 687 00000100:00100000:0.0:1767318723.529880:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982633728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318723.529918:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982633728:12345-0@lo:400:kworker.0 Request processed in 38us (251us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318723.529924:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 687 00010000:00080000:0.0:1767318723.529958:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57800 x1853163982633728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318739 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318723.529968:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982633728:0@lo:400:kworker.0 00000100:00080000:0.0:1767318723.530017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318723.532943:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982634624:12345-0@lo:38:kworker.0 Request processed in 3467us (3517us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318723.532959:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 897 00000100:00080000:1.0:1767318723.533771:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318723.533779:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318723.533783:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982634624:0@lo:38:kworker.0 00000100:00080000:1.0:1767318723.533799:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318723.533801:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318723.533804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982634112:0@lo:38:kworker.0 00000100:00080000:1.0:1767318723.533813:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318723.533817:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318723.533819:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982634240:0@lo:38:kworker.0 00000100:00100000:3.0:1767318723.536735:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318723.547154:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634496 00000100:00100000:0.0:1767318723.547175:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634496 00000100:00100000:0.0:1767318723.547307:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 450 00000100:00100000:0.0:1767318723.547322:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982634496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318723.547437:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982634496:12345-0@lo:400:kworker.0 Request processed in 120us (10709us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318723.547445:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 450 00000100:00100000:1.0:1767318723.547657:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982634496:0@lo:400:kworker.0 00000100:00080000:2.0:1767318723.547688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318723.716510:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318724.740528:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318725.515745:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318725.516037:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584768 00000100:00100000:2.0:1767318725.516050:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584768 00000100:00100000:2.0:1767318725.516104:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 898 00000100:00100000:2.0:1767318725.516112:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584768:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318725.516135:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318725.516737:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915584768:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 623us (1000us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318725.516748:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 898 00000100:00100000:0.0:1767318725.520172:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318725.521054:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915584896 00000100:00100000:2.0:1767318725.521067:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915584896 00000100:00100000:2.0:1767318725.521160:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 688 00000100:00100000:2.0:1767318725.521171:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915584896:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318725.521258:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915584896:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (1108us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318725.521265:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 688 00000100:00080000:0.0:1767318725.522133:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318725.766708:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318726.788576:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318727.108609:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982634752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318727.108683:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318727.108902:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634752 00000100:00100000:1.0:1767318727.108911:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634752 00000100:00100000:0.0:1767318727.108953:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318727.108966:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982634880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318727.108996:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 480 00000100:00100000:1.0:1767318727.109008:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982634752:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318727.109036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:1.0:1767318727.109040:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318727.109073:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982634880 00000100:00100000:0.0:1767318727.109078:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982634880 00000100:00100000:1.0:1767318727.109097:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982634752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (412us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318727.109104:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 481 00000100:00100000:0.0:1767318727.109112:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982634880:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318727.109206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982634752:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318727.109284:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318727.109289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:0.0:1767318727.109348:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318727.109388:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982634880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 276us (352us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318727.109439:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982634880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318727.110261:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 480 00000100:00100000:0.0:1767318727.115542:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 481 00010000:00010000:1.0:1767318727.813787:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318728.644773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318728.644783:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318728.644846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318728.644858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318728.644862:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318728.644870:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3972 00000100:00080000:0.0:1767318728.644875:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318728.644881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318728.644903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4e00 x1853163982635136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318728.644978:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982635136:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.645047:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318728.645137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982635008:0@lo:400:kworker.0 00000100:00100000:2.0:1767318728.645168:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318728.645204:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635136 00000100:00100000:2.0:1767318728.645212:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635136 00000100:00100000:2.0:1767318728.645244:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635008 00000100:00100000:2.0:1767318728.645246:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635008 00000100:00100000:2.0:1767318728.645312:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 689 00000100:00100000:2.0:1767318728.645320:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982635008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318728.645363:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982635008:12345-0@lo:400:kworker.0 Request processed in 44us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318728.645369:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 689 00000100:00100000:2.0:1767318728.645384:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 899 00000100:00100000:2.0:1767318728.645388:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982635136:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767318728.645596:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:2.0:1767318728.645650:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318728.645731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318728.645734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318728.645736:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318728.645739:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3972 00000100:00080000:0.0:1767318728.645742:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318728.645746:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318728.645759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163982635392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318728.646573:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982635264:0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.646614:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.646658:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635264 00000100:00100000:1.0:1767318728.646663:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635264 00000100:00100000:1.0:1767318728.646700:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 690 00000100:00100000:1.0:1767318728.646706:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982635264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.646744:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982635264:12345-0@lo:400:kworker.0 Request processed in 37us (130us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318728.646751:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 690 00000100:00080000:1.0:1767318728.646770:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318728.646793:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163982635264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318744 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318728.646806:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982635264:0@lo:400:kworker.0 00010000:00080000:1.0:1767318728.646876:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6300 x1853163982635008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318744 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318728.646886:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982635008:0@lo:400:kworker.0 00000100:00100000:3.0:1767318728.647125:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982635392:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.647155:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318728.647215:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635392 00000100:00100000:1.0:1767318728.647220:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635392 00000100:00100000:1.0:1767318728.647238:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900 00000100:00100000:1.0:1767318728.647244:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982635392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318728.647259:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318728.647308:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982635392:12345-0@lo:38:kworker.0 Request processed in 62us (151us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318728.647315:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900 00000100:00100000:2.0:1767318728.649688:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982635136:12345-0@lo:38:kworker.0 Request processed in 4289us (4632us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318728.649701:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899 00000100:00080000:0.0:1767318728.652005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318728.652015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318728.652018:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318728.652025:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3972 00000100:00080000:0.0:1767318728.652030:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318728.652037:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318728.652080:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9500 x1853163982635520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318728.652156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318728.652160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318728.652161:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318728.652166:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3972 00000100:00080000:0.0:1767318728.652168:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318728.652172:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318728.652184:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163982635904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318728.652223:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982635776:0@lo:400:kworker.0 00000100:00100000:0.0:1767318728.652319:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318728.652413:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635776 00000100:00100000:0.0:1767318728.652420:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635776 00000100:00100000:0.0:1767318728.652463:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 451 00000100:00100000:0.0:1767318728.652472:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982635776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318728.652523:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982635776:12345-0@lo:400:kworker.0 Request processed in 50us (203us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318728.652530:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 451 00000100:00100000:0.0:1767318728.652575:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982635776:0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.652892:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982635648:0@lo:400:kworker.0 00000100:00100000:1.0:1767318728.652923:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318728.653854:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635648 00000100:00100000:0.0:1767318728.653862:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635648 00000100:00100000:0.0:1767318728.653908:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 452 00000100:00100000:0.0:1767318728.653916:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982635648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318728.653988:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982635648:12345-0@lo:400:kworker.0 Request processed in 72us (1064us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318728.653995:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 452 00000100:00100000:2.0:1767318728.654106:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982635648:0@lo:400:kworker.0 00000100:00080000:3.0:1767318728.654476:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318728.654482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318728.654485:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982635136:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.654508:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982635520:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.654538:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318728.654571:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982635904:0@lo:38:kworker.0 00000100:00100000:3.0:1767318728.654591:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318728.654615:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318728.654618:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318728.654621:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982635392:0@lo:38:kworker.0 00000100:00100000:2.0:1767318728.654678:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635904 00000100:00100000:2.0:1767318728.654682:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635904 00000100:00100000:2.0:1767318728.654717:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 901 00000100:00100000:2.0:1767318728.654723:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982635904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318728.654741:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318728.654779:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982635904:12345-0@lo:38:kworker.0 Request processed in 55us (187us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318728.654788:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 901 00000100:00080000:3.0:1767318728.654967:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318728.654972:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318728.654975:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982635904:0@lo:38:kworker.0 00000100:00100000:0.0:1767318728.655377:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982635520 00000100:00100000:0.0:1767318728.655382:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982635520 00000100:00100000:0.0:1767318728.655525:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 250 00000100:00100000:0.0:1767318728.655530:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982635520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318728.655543:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318728.655588:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982635520:12345-0@lo:38:kworker.0 Request processed in 56us (1049us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318728.655594:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 250 00000100:00080000:3.0:1767318728.655644:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318728.655649:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318728.655652:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982635520:0@lo:38:kworker.0 00010000:00010000:1.0:1767318728.838646:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318729.861046:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318730.637198:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318730.637250:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318730.637351:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585152 00000100:00100000:2.0:1767318730.637371:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585152 00000100:00100000:2.0:1767318730.637436:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585024 00000100:00100000:2.0:1767318730.637439:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585024 00000100:00100000:2.0:1767318730.637455:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 691 00000100:00100000:2.0:1767318730.637464:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915585152:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318730.637539:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915585152:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (351us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318730.637547:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 691 00000100:00100000:2.0:1767318730.637565:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 902 00000100:00100000:2.0:1767318730.637569:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915585024:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318730.637589:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318730.637615:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915585024:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (365us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318730.637622:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 902 00000100:00080000:1.0:1767318730.638097:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318730.885546:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318731.908546:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318732.228635:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318732.228640:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982636032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318732.228655:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982636160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318732.231428:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318732.231807:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636032 00000100:00100000:0.0:1767318732.231916:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636032 00000100:00080000:2.0:1767318732.232190:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318732.232239:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 482 00000100:00100000:0.0:1767318732.232250:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324e0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982636032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318732.232319:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318732.232371:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324e0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982636032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (950us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318732.232379:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 482 00000100:00100000:2.0:1767318732.233720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982636032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318732.235203:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318732.235676:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636160 00000100:00100000:0.0:1767318732.235687:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636160 00000100:00100000:0.0:1767318732.235782:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 483 00000100:00100000:0.0:1767318732.235793:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982636160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318732.235827:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318732.235907:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982636160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (720us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318732.235915:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 483 00000100:00080000:2.0:1767318732.235997:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318732.236221:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982636160:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318732.932927:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318733.764770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318733.764781:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318733.764861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318733.764865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318733.764869:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318733.764877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318733.764882:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318733.764889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318733.764911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1880 x1853163982636416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318733.764958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318733.764961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318733.764963:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318733.764967:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318733.764970:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318733.764974:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318733.764988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7b80 x1853163982636672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318733.764997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318733.765000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318733.765002:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318733.765006:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318733.765008:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318733.765011:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318733.765022:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5500 x1853163982636800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318733.765041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318733.765045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318733.765047:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318733.765050:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3977 00000100:00080000:1.0:1767318733.765052:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318733.765055:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318733.765065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163982637184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318733.765092:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982636544:0@lo:400:kworker.0 00000100:00100000:1.0:1767318733.765159:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318733.765363:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982637056:0@lo:400:kworker.0 00000100:00100000:3.0:1767318733.765401:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318733.765565:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318733.765571:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982636288:0@lo:400:kworker.0 00000100:00100000:3.0:1767318733.765798:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637056 00000100:00100000:3.0:1767318733.765807:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637056 00000100:00100000:3.0:1767318733.765871:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 453 00000100:00100000:3.0:1767318733.765882:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982637056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318733.765996:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982637056:12345-0@lo:400:kworker.0 Request processed in 116us (593us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318733.766004:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 453 00000100:00100000:2.0:1767318733.766133:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982636928:0@lo:400:kworker.0 00000100:00100000:2.0:1767318733.766166:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318733.766538:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636928 00000100:00100000:0.0:1767318733.766545:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636928 00000100:00100000:0.0:1767318733.766578:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 454 00000100:00100000:0.0:1767318733.766598:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982636928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318733.766643:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982636928:12345-0@lo:400:kworker.0 Request processed in 59us (476us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318733.766821:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 454 00000100:00100000:0.0:1767318733.766860:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982636416:0@lo:38:kworker.0 00000100:00100000:0.0:1767318733.766892:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318733.766918:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982636672:0@lo:38:kworker.0 00000100:00100000:0.0:1767318733.766942:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318733.766958:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982636800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318733.766981:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318733.767001:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982637184:0@lo:38:kworker.0 00000100:00100000:0.0:1767318733.767019:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318733.767044:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636544 00000100:00100000:0.0:1767318733.767049:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636544 00000100:00100000:0.0:1767318733.767079:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636416 00000100:00100000:0.0:1767318733.767083:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636416 00000100:00100000:0.0:1767318733.767090:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636672 00000100:00100000:0.0:1767318733.767092:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636672 00000100:00100000:0.0:1767318733.767098:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637184 00000100:00100000:0.0:1767318733.767100:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637184 00000100:00100000:0.0:1767318733.767109:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 692 00000100:00100000:0.0:1767318733.767115:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982636544:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318733.767149:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982636544:12345-0@lo:400:kworker.0 Request processed in 34us (1990us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318733.767154:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 692 00000100:00100000:0.0:1767318733.767166:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 903 00000100:00100000:3.0:1767318733.767429:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318733.767475:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982637056:0@lo:400:kworker.0 00000100:00080000:3.0:1767318733.767509:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318733.767533:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636288 00000100:00100000:3.0:1767318733.767538:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636288 00000100:00100000:0.0:1767318733.767763:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982636416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318733.767783:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318733.767819:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982636416:12345-0@lo:38:kworker.0 Request processed in 56us (926us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318733.776170:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 904 00000100:00100000:1.0:1767318733.776191:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982636672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318733.776216:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318733.776335:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982636672:12345-0@lo:38:kworker.0 Request processed in 148us (9392us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318733.776343:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 904 00000100:00100000:1.0:1767318733.776378:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905 00000100:00100000:1.0:1767318733.776384:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982637184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318733.776394:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318733.776422:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982637184:12345-0@lo:38:kworker.0 Request processed in 36us (9401us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318733.776428:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905 00010000:00080000:1.0:1767318733.776491:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3480 x1853163982636544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318749 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318733.776509:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982636544:0@lo:400:kworker.0 00000100:00080000:1.0:1767318733.776596:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318733.776602:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318733.776606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982636416:0@lo:38:kworker.0 00000100:00080000:1.0:1767318733.776618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318733.776621:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318733.776624:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982636672:0@lo:38:kworker.0 00000100:00080000:1.0:1767318733.776636:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318733.776639:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318733.776641:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982637184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318733.777777:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982636800 00000100:00100000:1.0:1767318733.777789:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982636800 00000100:00100000:1.0:1767318733.777842:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 251 00000100:00100000:1.0:1767318733.777860:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982636800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318733.777877:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318733.777941:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982636800:12345-0@lo:38:kworker.0 Request processed in 80us (10958us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318733.777947:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 251 00000100:00100000:1.0:1767318733.777992:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982636928:0@lo:400:kworker.0 00000100:00080000:1.0:1767318733.778019:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318733.778023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318733.778026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982636800:0@lo:38:kworker.0 00000100:00080000:1.0:1767318733.778052:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318733.779087:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 693 00000100:00100000:3.0:1767318733.779104:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982636288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318733.779346:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982636288:12345-0@lo:400:kworker.0 Request processed in 249us (11915us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318733.779354:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 693 00000100:00100000:0.0:1767318733.779601:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 903 00010000:00080000:1.0:1767318733.779821:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2300 x1853163982636288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318749 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318733.779835:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982636288:0@lo:400:kworker.0 00010000:00010000:1.0:1767318733.956597:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767318734.734411:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915585280 offset 0 length 520: 4 00000100:00100000:2.0:1767318734.734473:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318734.734591:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318734.735086:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585536 00000100:00100000:0.0:1767318734.735098:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585536 00000100:00100000:0.0:1767318734.735174:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585408 00000100:00100000:0.0:1767318734.735178:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585408 00000100:00100000:0.0:1767318734.735216:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 694 00000100:00100000:0.0:1767318734.735229:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915585536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318734.735353:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915585536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 126us (879us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318734.735361:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 694 00000100:00100000:0.0:1767318734.735384:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 906 00000100:00100000:0.0:1767318734.735390:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915585408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318734.735411:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318734.735454:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915585408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 62us (863us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318734.735461:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 906 00000100:00080000:1.0:1767318734.736124:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767318734.981265:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318736.006404:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318737.028752:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318737.348654:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318737.348682:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982637312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318737.348758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318737.349345:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637312 00000100:00100000:0.0:1767318737.349355:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637312 00000100:00100000:0.0:1767318737.349432:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 484 00000100:00100000:0.0:1767318737.349441:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982637312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318737.349470:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318737.349535:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982637312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (776us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318737.349542:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 484 00000100:00100000:0.0:1767318737.349592:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982637312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318737.349738:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982637440:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318737.349771:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318737.349909:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637440 00000100:00100000:0.0:1767318737.349914:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637440 00000100:00100000:0.0:1767318737.349940:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 485 00000100:00100000:0.0:1767318737.349947:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982637440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318737.349964:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318737.349994:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982637440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318737.350000:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 485 00000100:00080000:1.0:1767318737.350351:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318737.350356:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318737.351047:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982637440:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767318738.052574:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318738.884470:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318713/real 1767318713] req@ffff93463c624000 x1853163982631040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318738 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318738.884513:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c624000 x1853163982631040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318738 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318738.884536:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318738.884566:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318738.884571:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982631040:0@lo:250:kworker.0 00000100:00080000:0.0:1767318738.884670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318738.884676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318738.884680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318738.884686:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3961 00000100:00080000:0.0:1767318738.884690:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318738.884698:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318738.884732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982637568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318738.884798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318738.884801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318738.884803:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318738.884807:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3982 00000100:00080000:0.0:1767318738.884809:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318738.884813:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318738.884827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163982637824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318738.884855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318738.884859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318738.884861:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318738.884864:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3982 00000100:00080000:0.0:1767318738.884867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318738.884870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318738.884880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982638080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318738.884888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318738.884890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318738.884892:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318738.884895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3982 00000100:00080000:0.0:1767318738.884897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318738.884899:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318738.884908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1c00 x1853163982638208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318738.884927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318738.884931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318738.884932:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318738.884935:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3982 00000100:00080000:0.0:1767318738.884937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318738.884940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318738.884949:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246300 x1853163982638592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318738.884974:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982637952:0@lo:400:kworker.0 00000100:00100000:0.0:1767318738.885047:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318738.885104:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318738.885110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982638336:0@lo:400:kworker.0 00000100:00100000:0.0:1767318738.885138:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318738.885177:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637952 00000100:00100000:0.0:1767318738.885184:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637952 00000100:00100000:0.0:1767318738.885249:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 695 00000100:00100000:0.0:1767318738.885259:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982637952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318738.885345:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982637952:12345-0@lo:400:kworker.0 Request processed in 87us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318738.885353:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 695 00000100:00100000:0.0:1767318738.885377:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638336 00000100:00100000:0.0:1767318738.885381:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638336 00000100:00100000:0.0:1767318738.885404:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 455 00000100:00100000:0.0:1767318738.885410:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982638336:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318738.885440:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982638336:12345-0@lo:400:kworker.0 Request processed in 30us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318738.885446:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 455 00000100:00100000:0.0:1767318738.885503:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982638336:0@lo:400:kworker.0 00010000:00080000:0.0:1767318738.885513:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0380 x1853163982637952/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318738.885523:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982637952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318738.885777:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982637568:0@lo:250:kworker.0 00000400:00000100:2.0:1767318738.885808:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982637568 offset 0 length 520: 4 00000100:00100000:2.0:1767318738.885830:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982637824:0@lo:38:kworker.0 00000100:00100000:2.0:1767318738.885855:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318738.885880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982638080:0@lo:38:kworker.0 00000100:00100000:2.0:1767318738.885899:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318738.885914:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982638208:0@lo:38:kworker.0 00000100:00100000:2.0:1767318738.885935:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318738.885947:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982638592:0@lo:38:kworker.0 00000100:00100000:2.0:1767318738.885963:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318738.885990:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637824 00000100:00100000:2.0:1767318738.885994:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637824 00000100:00100000:2.0:1767318738.886014:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638080 00000100:00100000:2.0:1767318738.886017:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638080 00000100:00100000:2.0:1767318738.886026:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638592 00000100:00100000:2.0:1767318738.886028:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638592 00000100:00100000:2.0:1767318738.886036:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 907 00000100:00100000:2.0:1767318738.886041:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982637824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318738.886061:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318738.886092:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982637824:12345-0@lo:38:kworker.0 Request processed in 49us (236us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318738.886099:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 907 00000100:00100000:2.0:1767318738.886111:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908 00000100:00100000:2.0:1767318738.886115:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982638080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318738.886124:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318738.886146:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982638080:12345-0@lo:38:kworker.0 Request processed in 29us (245us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318738.886151:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908 00000100:00100000:2.0:1767318738.886161:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909 00000100:00100000:2.0:1767318738.886167:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982638592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318738.886175:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318738.886196:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982638592:12345-0@lo:38:kworker.0 Request processed in 27us (232us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318738.886213:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909 00000100:00080000:2.0:1767318738.886243:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318738.886248:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318738.886251:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982637824:0@lo:38:kworker.0 00000100:00080000:2.0:1767318738.886258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318738.886261:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318738.886264:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982638080:0@lo:38:kworker.0 00000100:00080000:2.0:1767318738.886295:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318738.886299:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318738.886301:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982638592:0@lo:38:kworker.0 00000100:00100000:3.0:1767318738.886427:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982638464:0@lo:400:kworker.0 00000100:00100000:3.0:1767318738.886470:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318738.886499:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318738.886505:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982637696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318738.886526:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318738.886656:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638464 00000100:00100000:0.0:1767318738.886661:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638464 00000100:00100000:0.0:1767318738.886696:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 456 00000100:00100000:0.0:1767318738.886702:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982638464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318738.886741:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982638464:12345-0@lo:400:kworker.0 Request processed in 39us (271us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318738.886759:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 456 00000100:00080000:1.0:1767318738.887019:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318738.887033:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638208 00000100:00100000:1.0:1767318738.887038:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638208 00000100:00100000:1.0:1767318738.887061:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 252 00000100:00100000:1.0:1767318738.887066:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982638208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318738.887077:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318738.887115:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982638208:12345-0@lo:38:kworker.0 Request processed in 48us (1179us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318738.887122:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 252 00000100:00100000:1.0:1767318738.887139:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982637696 00000100:00100000:1.0:1767318738.887142:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982637696 00000100:00100000:1.0:1767318738.887170:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00100000:1.0:1767318738.887176:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982637696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318738.887219:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982637696:12345-0@lo:400:kworker.0 Request processed in 43us (691us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318738.887226:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00080000:1.0:1767318738.887239:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318738.887258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318738.887262:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318738.887265:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982638208:0@lo:38:kworker.0 00010000:00080000:3.0:1767318738.889957:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163982637696/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318754 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318738.889975:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982637696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318738.889991:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982638464:0@lo:400:kworker.0 00010000:00010000:1.0:1767318739.076666:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318739.853792:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318739.853843:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318739.853952:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585792 00000100:00100000:2.0:1767318739.853963:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585792 00000100:00080000:3.0:1767318739.854038:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318739.854048:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585664 00000100:00100000:2.0:1767318739.854053:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585664 00000100:00100000:2.0:1767318739.854077:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 697 00000100:00100000:2.0:1767318739.854087:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915585792:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318739.854170:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915585792:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (388us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318739.854178:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 697 00000100:00100000:1.0:1767318739.854184:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 910 00000100:00100000:1.0:1767318739.854190:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915585664:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318739.854213:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318739.854242:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915585664:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (399us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318739.854251:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 910 00010000:00010000:1.0:1767318740.100720:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318741.129784:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318742.152614:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318742.469015:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982638720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318742.469095:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318742.469237:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638720 00000100:00100000:1.0:1767318742.469246:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638720 00000100:00100000:1.0:1767318742.469326:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 486 00000100:00100000:1.0:1767318742.469336:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982638720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318742.469367:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318742.469414:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982638720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (318us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318742.469422:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 486 00000100:00080000:2.0:1767318742.469644:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318742.469708:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982638720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318742.471336:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318742.471352:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982638848:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318742.471405:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318742.471452:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638848 00000100:00100000:3.0:1767318742.471459:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638848 00000100:00100000:3.0:1767318742.471511:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 487 00000100:00100000:3.0:1767318742.471519:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982638848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318742.471545:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318742.471590:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982638848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (184us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318742.471598:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 487 00000100:00100000:3.0:1767318742.471637:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982638848:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318742.471774:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318743.172872:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318744.005535:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318744.005545:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318744.005643:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982638976:0@lo:400:kworker.0 00000100:00100000:2.0:1767318744.005731:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318744.005804:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318744.005817:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318744.005821:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318744.005826:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3987 00000100:00080000:3.0:1767318744.005830:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318744.005837:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318744.005863:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163982639104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318744.005901:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318744.005905:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318744.005907:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318744.005911:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3987 00000100:00080000:3.0:1767318744.005914:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318744.005918:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318744.005929:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4e00 x1853163982639360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318744.005939:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318744.005942:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318744.005944:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318744.005957:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3987 00000100:00080000:3.0:1767318744.005961:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318744.005964:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318744.005979:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163982639488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318744.006046:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982639744:0@lo:400:kworker.0 00000100:00100000:1.0:1767318744.006081:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318744.006138:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639744 00000100:00100000:1.0:1767318744.006146:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639744 00000100:00100000:1.0:1767318744.006195:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 457 00000100:00100000:1.0:1767318744.006204:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982639744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318744.006257:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982639744:12345-0@lo:400:kworker.0 Request processed in 53us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318744.006264:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 457 00000100:00100000:1.0:1767318744.006311:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982639232:0@lo:400:kworker.0 00000100:00100000:1.0:1767318744.006338:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318744.006562:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318744.006569:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982639616:0@lo:400:kworker.0 00000100:00100000:1.0:1767318744.006594:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318744.009726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318744.009757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982639104:0@lo:38:kworker.0 00000100:00100000:2.0:1767318744.009847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318744.010029:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318744.010036:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318744.010040:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318744.010045:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3987 00000100:00080000:3.0:1767318744.010049:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318744.010056:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318744.010087:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163982639872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318744.010167:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982639744:0@lo:400:kworker.0 00000100:00100000:3.0:1767318744.010197:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639616 00000100:00100000:3.0:1767318744.010203:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639616 00000100:00100000:3.0:1767318744.010263:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 458 00000100:00100000:3.0:1767318744.010286:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982639616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318744.010877:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982639616:12345-0@lo:400:kworker.0 Request processed in 76us (3751us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318744.010887:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 458 00000100:00080000:3.0:1767318744.010923:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318744.010986:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982639616:0@lo:400:kworker.0 00000100:00100000:2.0:1767318744.012099:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982639360:0@lo:38:kworker.0 00000100:00100000:2.0:1767318744.012155:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318744.014732:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982638976 00000100:00100000:0.0:1767318744.014745:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982638976 00000100:00080000:0.0:1767318744.014796:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318744.014869:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639232 00000100:00100000:0.0:1767318744.014873:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639232 00000100:00100000:0.0:1767318744.014902:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639104 00000100:00100000:0.0:1767318744.014906:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639104 00000100:00100000:0.0:1767318744.014922:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767318744.014933:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982638976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318744.015030:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982638976:12345-0@lo:400:kworker.0 Request processed in 98us (9298us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318744.015038:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767318744.015055:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000100:00100000:0.0:1767318744.015062:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982639232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318744.015097:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982639232:12345-0@lo:400:kworker.0 Request processed in 36us (8758us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318744.015104:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000100:00100000:0.0:1767318744.015116:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 911 00000100:00100000:0.0:1767318744.015121:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982639104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318744.015138:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318744.015167:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982639104:12345-0@lo:38:kworker.0 Request processed in 45us (5320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318744.015174:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 911 00000100:00080000:0.0:1767318744.015191:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318744.015227:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee7b80 x1853163982639232/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318760 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318744.015243:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982639232:0@lo:400:kworker.0 00010000:00080000:0.0:1767318744.015303:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee4380 x1853163982638976/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318760 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318744.015313:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982638976:0@lo:400:kworker.0 00000100:00100000:2.0:1767318744.020054:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982639488:0@lo:38:kworker.0 00000100:00100000:2.0:1767318744.020122:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318744.020155:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982639872:0@lo:38:kworker.0 00000100:00100000:2.0:1767318744.020182:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318744.020220:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318744.020226:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318744.020230:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982639104:0@lo:38:kworker.0 00000100:00100000:0.0:1767318744.020358:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639488 00000100:00100000:0.0:1767318744.020367:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639488 00000100:00100000:0.0:1767318744.020406:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 253 00000100:00100000:0.0:1767318744.020413:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982639488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318744.020434:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318744.020476:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982639488:12345-0@lo:38:kworker.0 Request processed in 62us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318744.020484:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 253 00000100:00100000:0.0:1767318744.020508:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639360 00000100:00100000:0.0:1767318744.020512:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639360 00000100:00100000:0.0:1767318744.020535:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982639872 00000100:00100000:0.0:1767318744.020538:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982639872 00000100:00100000:0.0:1767318744.020556:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912 00000100:00100000:0.0:1767318744.020560:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982639360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318744.020570:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318744.020596:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982639360:12345-0@lo:38:kworker.0 Request processed in 35us (8441us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318744.020606:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912 00000100:00100000:0.0:1767318744.020618:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 913 00000100:00100000:0.0:1767318744.020622:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982639872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318744.020631:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318744.020664:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982639872:12345-0@lo:38:kworker.0 Request processed in 30us (471us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318744.020670:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 913 00000100:00080000:2.0:1767318744.022120:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318744.022129:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318744.022133:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982639872:0@lo:38:kworker.0 00000100:00080000:2.0:1767318744.022145:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318744.022149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318744.022152:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982639360:0@lo:38:kworker.0 00000100:00080000:2.0:1767318744.022161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318744.022164:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318744.022167:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982639488:0@lo:38:kworker.0 00010000:00010000:3.0:1767318744.196658:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318744.972825:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318744.972925:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318744.973178:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586048 00000100:00100000:2.0:1767318744.973189:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586048 00000100:00100000:2.0:1767318744.973265:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915585920 00000100:00100000:2.0:1767318744.973285:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915585920 00000100:00100000:2.0:1767318744.973305:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 700 00000100:00100000:2.0:1767318744.973315:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915586048:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318744.973390:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915586048:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (577us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318744.973398:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 700 00000100:00100000:2.0:1767318744.973416:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 914 00000100:00100000:2.0:1767318744.973421:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915585920:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318744.973440:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318744.973465:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915585920:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (541us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318744.973472:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 914 00000100:00080000:0.0:1767318744.973815:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318745.223443:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318746.246765:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318747.281015:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318747.589554:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982640000:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318747.589639:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318747.589771:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640000 00000100:00100000:1.0:1767318747.589780:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640000 00000100:00100000:1.0:1767318747.589842:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 488 00000100:00100000:1.0:1767318747.589851:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982640000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318747.589881:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318747.589930:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982640000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (290us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318747.589936:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 488 00000100:00100000:1.0:1767318747.589981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982640000:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318747.591003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318747.591396:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982640128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318747.591441:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318747.591582:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640128 00000100:00100000:3.0:1767318747.591598:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640128 00000100:00100000:3.0:1767318747.591655:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 489 00000100:00100000:3.0:1767318747.591664:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982640128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318747.591687:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318747.591733:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982640128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (291us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318747.591741:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 489 00000100:00100000:3.0:1767318747.591790:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982640128:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318747.592788:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318748.295186:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318749.124411:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318749.124425:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318749.124486:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318749.124490:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318749.124494:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318749.124503:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3992 00000100:00080000:3.0:1767318749.124508:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318749.124515:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318749.124538:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163982640384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318749.124635:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318749.124639:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318749.124641:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318749.124646:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3992 00000100:00080000:3.0:1767318749.124649:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318749.124653:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318749.124667:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8380 x1853163982640640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318749.124676:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318749.124679:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318749.124681:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318749.124684:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3992 00000100:00080000:3.0:1767318749.124687:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318749.124690:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318749.124702:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982640768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318749.124726:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318749.124730:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318749.124732:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318749.124735:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3992 00000100:00080000:3.0:1767318749.124738:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318749.124741:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318749.124752:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982641152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318749.124793:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982640512:0@lo:400:kworker.0 00000100:00100000:3.0:1767318749.124859:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318749.124903:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318749.124910:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982640896:0@lo:400:kworker.0 00000100:00100000:3.0:1767318749.124946:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318749.124980:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640512 00000100:00100000:3.0:1767318749.124989:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640512 00000100:00100000:3.0:1767318749.125434:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 701 00000100:00100000:3.0:1767318749.125446:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982640512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318749.125501:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982640512:12345-0@lo:400:kworker.0 Request processed in 58us (641us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318749.125509:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 701 00000100:00080000:3.0:1767318749.125531:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318749.125542:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640896 00000100:00100000:3.0:1767318749.125546:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640896 00000100:00100000:3.0:1767318749.125592:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 459 00000100:00100000:1.0:1767318749.125602:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982640384:0@lo:38:kworker.0 00000100:00100000:3.0:1767318749.125602:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982640896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318749.125627:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982640256:0@lo:400:kworker.0 00000100:00100000:1.0:1767318749.125653:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318749.125677:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640384 00000100:00100000:2.0:1767318749.125680:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640384 00000100:00100000:1.0:1767318749.125681:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982640640:0@lo:38:kworker.0 00000100:00100000:2.0:1767318749.125702:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915 00000100:00100000:1.0:1767318749.125703:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318749.125708:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982640384:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318749.125719:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982640768:0@lo:38:kworker.0 00010000:00020000:2.0:1767318749.125729:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318749.125756:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318749.125758:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982640384:12345-0@lo:38:kworker.0 Request processed in 50us (106us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318749.125764:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915 00000100:00100000:2.0:1767318749.125773:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640640 00000100:00100000:2.0:1767318749.125776:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640640 00000100:00100000:1.0:1767318749.125779:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982641152:0@lo:38:kworker.0 00000100:00100000:2.0:1767318749.125789:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916 00000100:00100000:2.0:1767318749.125793:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982640640:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318749.125799:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767318749.125802:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318749.125822:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982640640:12345-0@lo:38:kworker.0 Request processed in 28us (119us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318749.125826:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916 00000100:00080000:1.0:1767318749.125827:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318749.125831:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318749.125834:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641152 00000100:00100000:1.0:1767318749.125835:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982640384:0@lo:38:kworker.0 00000100:00100000:2.0:1767318749.125837:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641152 00000100:00080000:1.0:1767318749.125843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767318749.125846:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917 00000100:00080000:1.0:1767318749.125847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318749.125850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982640640:0@lo:38:kworker.0 00000100:00100000:2.0:1767318749.125850:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982641152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318749.125858:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:1.0:1767318749.125914:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9f80 x1853163982640512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318765 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318749.125926:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982640512:0@lo:400:kworker.0 00000100:00100000:1.0:1767318749.126447:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318749.126461:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982641152:12345-0@lo:38:kworker.0 Request processed in 608us (659us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318749.126468:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917 00000100:00100000:1.0:1767318749.126476:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318749.126482:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640256 00000100:00100000:1.0:1767318749.126487:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982641024:0@lo:400:kworker.0 00000100:00100000:2.0:1767318749.126488:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640256 00000100:00100000:0.0:1767318749.126509:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982640768 00000100:00100000:0.0:1767318749.126514:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982640768 00000100:00100000:1.0:1767318749.126523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318749.126543:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 254 00000100:00100000:0.0:1767318749.126550:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982640768:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318749.126554:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641024 00000100:00100000:1.0:1767318749.126568:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641024 00010000:00020000:0.0:1767318749.126571:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318749.126604:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 460 00000100:00100000:0.0:1767318749.126610:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982640768:12345-0@lo:38:kworker.0 Request processed in 59us (852us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318749.126615:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982641024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318749.126620:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 254 00000100:00100000:2.0:1767318749.126636:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000100:00100000:2.0:1767318749.126643:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982640256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318749.126677:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982640256:12345-0@lo:400:kworker.0 Request processed in 35us (231us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318749.126683:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00010000:00080000:0.0:1767318749.126689:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fad80 x1853163982640256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318765 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318749.126699:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982640256:0@lo:400:kworker.0 00000100:00080000:2.0:1767318749.126702:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318749.126706:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318749.126708:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982640768:0@lo:38:kworker.0 00000100:00080000:2.0:1767318749.126715:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318749.126719:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318749.126722:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982641152:0@lo:38:kworker.0 00000100:00080000:2.0:1767318749.126743:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318749.126746:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318749.126748:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318749.127180:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982641024:12345-0@lo:400:kworker.0 Request processed in 568us (656us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318749.127187:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 460 00000100:00100000:0.0:1767318749.127261:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982641024:0@lo:400:kworker.0 00000100:00100000:3.0:1767318749.127606:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982640896:12345-0@lo:400:kworker.0 Request processed in 2005us (2657us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318749.133023:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 459 00000100:00100000:0.0:1767318749.133685:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982640896:0@lo:400:kworker.0 00010000:00010000:1.0:1767318749.316741:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318750.096864:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318750.096923:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318750.097016:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586304 00000100:00100000:3.0:1767318750.097027:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586304 00000100:00100000:3.0:1767318750.097103:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586176 00000100:00100000:3.0:1767318750.097107:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586176 00000100:00100000:3.0:1767318750.097133:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 703 00000100:00100000:3.0:1767318750.097145:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915586304:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318750.097905:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 918 00000100:00100000:1.0:1767318750.097913:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915586176:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318750.097937:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318750.098022:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915586176:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 106us (1097us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318750.098030:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 918 00000100:00080000:2.0:1767318750.098508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318750.098692:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915586304:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1547us (1836us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318750.098702:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 703 00010000:00010000:3.0:1767318750.340737:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318751.365963:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318752.390291:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318752.709468:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982641280:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318752.709562:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318752.709751:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641280 00000100:00100000:1.0:1767318752.709763:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641280 00000100:00100000:0.0:1767318752.709783:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982641408:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318752.709809:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318752.709856:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318752.713541:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641408 00000100:00100000:1.0:1767318752.713559:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641408 00000100:00100000:1.0:1767318752.713628:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 490 00000100:00100000:1.0:1767318752.713638:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982641280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318752.713677:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318752.713761:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982641280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 124us (4199us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318752.713768:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 490 00000100:00100000:1.0:1767318752.713785:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 491 00000100:00100000:1.0:1767318752.713792:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982641408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318752.713807:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318752.713838:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982641408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (4028us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318752.713844:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 491 00000100:00100000:1.0:1767318752.713894:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982641408:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318752.713927:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982641280:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318752.715926:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:3.0:1767318753.413690:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318754.244555:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318754.244565:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318754.244634:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318754.244638:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318754.244641:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318754.244649:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 3997 00000100:00080000:2.0:1767318754.244654:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318754.244661:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318754.244683:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56d80 x1853163982641664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318754.244713:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318754.244716:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318754.244717:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318754.244721:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 3997 00000100:00080000:2.0:1767318754.244723:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318754.244726:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318754.244733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982641664:0@lo:38:kworker.0 00000100:00100000:2.0:1767318754.244737:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54700 x1853163982641920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318754.244745:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318754.244748:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318754.244749:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318754.244752:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 3997 00000100:00080000:2.0:1767318754.244754:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318754.244757:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318754.244765:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163982642048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318754.244795:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318754.244799:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318754.244800:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767318754.244801:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318754.244804:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 3997 00000100:00080000:2.0:1767318754.244806:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318754.244809:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318754.244818:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982642432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318754.244842:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982642176:0@lo:400:kworker.0 00000100:00100000:3.0:1767318754.245198:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982641536:0@lo:400:kworker.0 00000100:00100000:0.0:1767318754.245309:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982642304:0@lo:400:kworker.0 00000100:00100000:1.0:1767318754.247298:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982641920:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.247352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.247380:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982642048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.247405:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.247430:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982642432:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.247453:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.247511:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641664 00000100:00100000:1.0:1767318754.247519:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641664 00000100:00100000:3.0:1767318754.247540:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.247549:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641920 00000100:00100000:1.0:1767318754.247551:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641920 00000100:00100000:1.0:1767318754.247559:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642432 00000100:00100000:1.0:1767318754.247561:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642432 00000100:00100000:1.0:1767318754.247568:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641536 00000100:00100000:1.0:1767318754.247570:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641536 00000100:00100000:3.0:1767318754.247574:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982641792:0@lo:400:kworker.0 00000100:00100000:3.0:1767318754.247597:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.247600:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982641792 00000100:00100000:1.0:1767318754.247603:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982641792 00000100:00100000:3.0:1767318754.247624:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00100000:3.0:1767318754.247637:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982641536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318754.247693:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982641536:12345-0@lo:400:kworker.0 Request processed in 60us (151us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318754.249987:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318754.250068:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642176 00000100:00100000:2.0:1767318754.250075:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642176 00000100:00100000:2.0:1767318754.250142:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 461 00000100:00100000:2.0:1767318754.250151:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982642176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318754.250217:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982642176:12345-0@lo:400:kworker.0 Request processed in 68us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318754.250224:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 461 00000100:00100000:2.0:1767318754.250295:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982642176:0@lo:400:kworker.0 00010000:00080000:2.0:1767318754.250344:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163982641536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318770 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318754.250356:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982641536:0@lo:400:kworker.0 00000100:00080000:2.0:1767318754.250394:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318754.250399:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318754.250684:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318754.251898:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 705 00000100:00100000:1.0:1767318754.251915:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982641792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318754.252081:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982641792:12345-0@lo:400:kworker.0 Request processed in 172us (4481us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318754.252089:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 705 00000100:00100000:1.0:1767318754.252109:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 919 00000100:00100000:1.0:1767318754.252113:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982641664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318754.252133:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318754.252167:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982641664:12345-0@lo:38:kworker.0 Request processed in 52us (7365us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318754.252173:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 919 00000100:00100000:1.0:1767318754.252185:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767318754.252190:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982641920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318754.252198:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318754.252221:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982641920:12345-0@lo:38:kworker.0 Request processed in 31us (4870us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318754.252230:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767318754.252240:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921 00000100:00100000:1.0:1767318754.252244:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982642432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318754.252253:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318754.252292:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982642432:12345-0@lo:38:kworker.0 Request processed in 47us (4838us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318754.252298:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921 00000100:00080000:1.0:1767318754.252337:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318754.252343:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318754.252346:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982641920:0@lo:38:kworker.0 00000100:00080000:1.0:1767318754.252356:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318754.252360:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318754.252362:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982642432:0@lo:38:kworker.0 00000100:00080000:1.0:1767318754.252371:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318754.252374:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318754.252376:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982641664:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.252423:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642048 00000100:00100000:1.0:1767318754.252429:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642048 00000100:00100000:1.0:1767318754.252453:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 255 00000100:00100000:1.0:1767318754.252457:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982642048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318754.252479:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318754.252510:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982642048:12345-0@lo:38:kworker.0 Request processed in 51us (5104us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318754.252516:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 255 00000100:00080000:1.0:1767318754.252541:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318754.252544:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318754.252547:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982642048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318754.254642:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642304 00000100:00100000:1.0:1767318754.254656:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642304 00000100:00100000:1.0:1767318754.254737:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 462 00000100:00100000:1.0:1767318754.254747:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982642304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318754.254848:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982642304:12345-0@lo:400:kworker.0 Request processed in 103us (4164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318754.254855:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 462 00000100:00080000:1.0:1767318754.254875:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318754.254917:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982642304:0@lo:400:kworker.0 00000100:00100000:3.0:1767318754.255985:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00010000:00080000:3.0:1767318754.256046:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55880 x1853163982641792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318770 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318754.256058:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982641792:0@lo:400:kworker.0 00010000:00010000:1.0:1767318754.436752:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318755.212908:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318755.212953:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318755.213095:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586560 00000100:00100000:2.0:1767318755.213104:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586560 00000100:00100000:2.0:1767318755.213173:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586432 00000100:00100000:2.0:1767318755.213177:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586432 00000100:00100000:2.0:1767318755.213199:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 706 00000100:00100000:2.0:1767318755.213208:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915586560:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318755.213319:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915586560:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 112us (431us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318755.213327:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 706 00000100:00100000:2.0:1767318755.213344:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 922 00000100:00100000:2.0:1767318755.213348:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915586432:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318755.213366:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318755.213388:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915586432:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 38us (434us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318755.213406:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 922 00000100:00080000:1.0:1767318755.214338:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318755.464837:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767318756.484525:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318757.510728:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318757.832311:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318757.832336:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982642688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318757.832414:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318757.832651:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982642560:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318757.832682:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318757.832731:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642688 00000100:00100000:3.0:1767318757.832738:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642688 00000100:00100000:3.0:1767318757.832795:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642560 00000100:00100000:3.0:1767318757.832799:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642560 00000100:00100000:3.0:1767318757.832817:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 492 00000100:00100000:3.0:1767318757.832825:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982642688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318757.832853:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318757.832891:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982642688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (478us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318757.832899:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 492 00000100:00100000:3.0:1767318757.832914:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 493 00000100:00100000:3.0:1767318757.832920:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982642560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318757.832935:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318757.832961:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982642560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318757.832967:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 493 00000100:00100000:3.0:1767318757.833008:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982642560:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318757.833076:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318757.833081:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318757.833182:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982642688:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318758.533423:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318759.367591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318759.367603:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318759.367711:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982642816:0@lo:400:kworker.0 00000100:00100000:2.0:1767318759.367786:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318759.367932:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642816 00000100:00100000:3.0:1767318759.367941:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642816 00000100:00100000:3.0:1767318759.368017:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00080000:0.0:1767318759.368019:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318759.368047:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982642816:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318759.368266:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318759.368299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318759.368303:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318759.368309:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4002 00000100:00080000:1.0:1767318759.368313:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318759.368319:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318759.368344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2a00 x1853163982642944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318759.368399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318759.368402:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318759.368404:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318759.368408:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4002 00000100:00080000:1.0:1767318759.368411:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318759.368414:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318759.368425:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1500 x1853163982643200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318759.368434:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318759.368437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318759.368439:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318759.368442:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4002 00000100:00080000:1.0:1767318759.368444:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318759.368448:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318759.368463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982643328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318759.368582:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982642944:0@lo:38:kworker.0 00000100:00100000:0.0:1767318759.368632:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318759.368661:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982643200:0@lo:38:kworker.0 00000100:00100000:0.0:1767318759.368683:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318759.368699:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982643328:0@lo:38:kworker.0 00000100:00100000:0.0:1767318759.368721:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318759.368757:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982642944 00000100:00100000:0.0:1767318759.368762:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982642944 00000100:00100000:0.0:1767318759.368783:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643200 00000100:00100000:0.0:1767318759.368786:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643200 00000100:00100000:0.0:1767318759.368800:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767318759.368805:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982642944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318759.368824:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318759.368857:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982642944:12345-0@lo:38:kworker.0 Request processed in 50us (225us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318759.368864:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767318759.368879:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924 00000100:00100000:0.0:1767318759.368883:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982643200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318759.368892:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318759.368912:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982643200:12345-0@lo:38:kworker.0 Request processed in 28us (228us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318759.368918:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924 00000100:00100000:0.0:1767318759.368934:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643328 00000100:00100000:0.0:1767318759.368938:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643328 00000100:00100000:0.0:1767318759.368954:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 256 00000100:00100000:0.0:1767318759.368958:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982643328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318759.368967:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318759.368990:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982643328:12345-0@lo:38:kworker.0 Request processed in 31us (268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318759.368996:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 256 00000100:00080000:0.0:1767318759.369023:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318759.369027:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318759.369031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982642944:0@lo:38:kworker.0 00000100:00080000:0.0:1767318759.369038:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318759.369040:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318759.369043:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982643200:0@lo:38:kworker.0 00000100:00080000:0.0:1767318759.369049:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318759.369052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318759.369055:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982643328:0@lo:38:kworker.0 00000100:00100000:0.0:1767318759.369093:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982643072:0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.369114:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318759.369129:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318759.369134:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982643456:0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.369153:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318759.369175:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643456 00000100:00100000:0.0:1767318759.369179:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643456 00000100:00100000:0.0:1767318759.369209:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 463 00000100:00100000:0.0:1767318759.369216:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982643456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.369249:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982643456:12345-0@lo:400:kworker.0 Request processed in 33us (95us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318759.369255:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 463 00000100:00100000:0.0:1767318759.369300:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982643584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.369337:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318759.369360:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318759.369389:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982643456:0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.369404:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643584 00000100:00100000:0.0:1767318759.369407:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643584 00000100:00100000:0.0:1767318759.369431:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 464 00000100:00100000:0.0:1767318759.369437:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982643584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.370447:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982643584:12345-0@lo:400:kworker.0 Request processed in 1005us (1104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318759.370467:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 464 00000100:00080000:0.0:1767318759.370499:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318759.370524:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982643584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.370547:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643072 00000100:00100000:0.0:1767318759.370552:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643072 00000100:00100000:0.0:1767318759.370586:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00100000:0.0:1767318759.370591:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982643072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318759.370714:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982643072:12345-0@lo:400:kworker.0 Request processed in 122us (1599us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318759.370721:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00080000:0.0:1767318759.370735:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318759.370751:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0000 x1853163982643072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318775 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318759.370761:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982643072:0@lo:400:kworker.0 00000100:00100000:3.0:1767318759.372955:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982642816:12345-0@lo:400:kworker.0 Request processed in 4899us (5156us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318759.372974:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00080000:1.0:1767318759.375207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318759.375218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318759.375222:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318759.375232:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4002 00000100:00080000:1.0:1767318759.375237:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318759.375245:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318759.375300:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982643712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318759.376196:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982643712:0@lo:38:kworker.0 00000100:00100000:0.0:1767318759.376260:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318759.376703:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643712 00000100:00100000:3.0:1767318759.376714:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643712 00000100:00100000:3.0:1767318759.376756:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 925 00000100:00100000:3.0:1767318759.376762:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982643712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318759.376781:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318759.376952:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982643712:12345-0@lo:38:kworker.0 Request processed in 189us (690us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318759.376957:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 925 00000100:00080000:1.0:1767318759.377035:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318759.377039:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318759.377042:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982643712:0@lo:38:kworker.0 00010000:00080000:2.0:1767318759.378803:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3480 x1853163982642816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318775 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318759.378878:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982642816:0@lo:400:kworker.0 00010000:00010000:1.0:1767318759.556560:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318760.332992:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318760.333034:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318760.333060:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915586944 offset 0 length 520: 4 00000100:00100000:3.0:1767318760.333163:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586688 00000100:00100000:3.0:1767318760.333173:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586688 00000100:00100000:3.0:1767318760.333219:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915586816 00000100:00100000:3.0:1767318760.333223:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915586816 00000100:00100000:3.0:1767318760.333306:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 709 00000100:00100000:3.0:1767318760.333329:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915586816:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318760.333431:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915586816:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 117us (396us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318760.333439:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 709 00000100:00100000:3.0:1767318760.333457:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 926 00000100:00100000:3.0:1767318760.333462:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915586688:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318760.333479:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318760.333506:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915586688:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (526us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318760.333513:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 926 00000100:00080000:2.0:1767318760.334397:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318760.581813:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318761.604521:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318762.628781:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318762.948531:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318762.948556:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982643840:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318762.948639:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318762.948837:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643840 00000100:00100000:2.0:1767318762.948846:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643840 00000100:00100000:2.0:1767318762.948909:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 494 00000100:00100000:2.0:1767318762.948918:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982643840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318762.948960:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318762.949004:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982643840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318762.949011:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 494 00000100:00100000:1.0:1767318762.949095:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982643840:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318762.949639:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318762.949775:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982643968:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318762.949814:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318762.949898:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982643968 00000100:00100000:3.0:1767318762.949904:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982643968 00000100:00100000:3.0:1767318762.949965:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 495 00000100:00100000:3.0:1767318762.949974:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982643968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318762.949995:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318762.950038:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982643968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318762.950045:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 495 00000100:00100000:3.0:1767318762.950077:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982643968:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318762.950263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318763.460425:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318738/real 1767318738] req@ffff9346395e8000 x1853163982637568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318763 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318763.460460:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346395e8000 x1853163982637568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318763 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318763.460482:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.460508:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318763.460512:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982637568:0@lo:250:kworker.0 00000100:00080000:2.0:1767318763.460605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318763.460611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318763.460614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318763.460619:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 3987 00000100:00080000:2.0:1767318763.460623:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318763.460629:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318763.460662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56a00 x1853163982644096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318763.460719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767318763.460725:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982644224:0@lo:400:kworker.0 00000100:00080000:2.0:1767318763.460726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318763.460729:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318763.460736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4007 00000100:00080000:2.0:1767318763.460739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318763.460743:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318763.460760:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55880 x1853163982644352/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318763.460793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318763.460797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767318763.460802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318763.460802:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318763.460809:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4007 00000100:00080000:2.0:1767318763.460813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318763.460817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318763.460829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163982644608/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318763.460838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318763.460842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318763.460844:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318763.460848:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4007 00000100:00080000:2.0:1767318763.460850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318763.460854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318763.460866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56d80 x1853163982644736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318763.460890:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767318763.460890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318763.460896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318763.460898:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767318763.460902:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982644480:0@lo:400:kworker.0 00000100:00080000:2.0:1767318763.460904:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4007 00000100:00080000:2.0:1767318763.460909:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318763.460914:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318763.460928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982644864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318763.460938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318763.460971:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644224 00000100:00100000:0.0:1767318763.460977:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644224 00000100:00100000:2.0:1767318763.460982:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644480 00000100:00100000:2.0:1767318763.460987:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644480 00000100:00080000:3.0:1767318763.461088:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318763.461130:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000100:00100000:2.0:1767318763.461139:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982644480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318763.461202:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982644480:12345-0@lo:400:kworker.0 Request processed in 64us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318763.461210:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00000100:00100000:2.0:1767318763.461225:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000100:00100000:2.0:1767318763.461232:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982644224:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318763.461262:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982644224:12345-0@lo:400:kworker.0 Request processed in 30us (463us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318763.461268:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00010000:00080000:3.0:1767318763.461372:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54380 x1853163982644480/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318763.461386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982644480:0@lo:400:kworker.0 00000100:00100000:1.0:1767318763.461447:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982644096:0@lo:250:kworker.0 00000400:00000100:1.0:1767318763.461474:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982644096 offset 0 length 520: 4 00000100:00100000:1.0:1767318763.461497:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982644352:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.461518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318763.462236:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644352 00000100:00100000:0.0:1767318763.462244:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644352 00000100:00100000:1.0:1767318763.462256:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982644608:0@lo:38:kworker.0 00000100:00100000:0.0:1767318763.462286:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927 00000100:00100000:0.0:1767318763.462303:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982644352:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.462323:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:0.0:1767318763.462326:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.462340:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982644736:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.462382:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318763.462938:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982644352:12345-0@lo:38:kworker.0 Request processed in 633us (1416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318763.462945:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927 00000100:00100000:0.0:1767318763.462961:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644608 00000100:00100000:0.0:1767318763.462965:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644608 00000100:00100000:0.0:1767318763.462979:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 928 00000100:00100000:0.0:1767318763.462983:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982644608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318763.462996:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.464003:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982644864:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.464051:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318763.464087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.464092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318763.464095:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982644352:0@lo:38:kworker.0 00000100:00100000:1.0:1767318763.464175:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644864 00000100:00100000:1.0:1767318763.464181:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644864 00000100:00100000:1.0:1767318763.464203:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929 00000100:00100000:1.0:1767318763.464210:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982644864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318763.464229:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318763.464237:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982644608:12345-0@lo:38:kworker.0 Request processed in 1250us (1913us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318763.464244:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 928 00000100:00100000:1.0:1767318763.464267:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982644864:12345-0@lo:38:kworker.0 Request processed in 54us (214us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318763.464307:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644736 00000100:00100000:1.0:1767318763.464307:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1767318763.464313:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644736 00010000:00080000:1.0:1767318763.464351:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54e00 x1853163982644224/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318779 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318763.464364:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982644224:0@lo:400:kworker.0 00000100:00080000:1.0:1767318763.464391:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.464398:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318763.464402:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982644608:0@lo:38:kworker.0 00000100:00100000:0.0:1767318763.464406:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 257 00000100:00100000:0.0:1767318763.464412:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982644736:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767318763.464415:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318763.464421:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767318763.464425:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318763.464427:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982644864:0@lo:38:kworker.0 00000100:00100000:0.0:1767318763.464469:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982644736:12345-0@lo:38:kworker.0 Request processed in 55us (2085us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767318763.464474:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767318763.464479:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 257 00000100:00080000:1.0:1767318763.464482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318763.464488:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982644736:0@lo:38:kworker.0 00010000:00010000:1.0:1767318763.653105:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318764.676984:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318765.452940:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318765.452979:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318765.453034:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587072 00000100:00100000:0.0:1767318765.453043:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587072 00000100:00100000:0.0:1767318765.453079:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587200 00000100:00100000:0.0:1767318765.453081:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587200 00000100:00100000:0.0:1767318765.453124:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 712 00000100:00100000:0.0:1767318765.453135:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915587200:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318765.453211:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915587200:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (230us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318765.453221:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 712 00000100:00100000:0.0:1767318765.453239:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 930 00000100:00100000:0.0:1767318765.453243:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915587072:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318765.453305:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318765.453332:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915587072:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (401us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318765.453338:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 930 00000100:00080000:2.0:1767318765.453392:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318765.700555:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318766.725329:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318767.748749:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318768.068564:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318768.068590:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982644992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318768.068677:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318768.068841:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982645120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318768.068872:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318768.068968:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982644992 00000100:00100000:2.0:1767318768.068976:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982644992 00000100:00100000:2.0:1767318768.069040:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645120 00000100:00100000:2.0:1767318768.069043:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645120 00000100:00100000:2.0:1767318768.069064:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 496 00000100:00100000:2.0:1767318768.069072:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982644992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318768.069103:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318768.069149:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982644992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318768.069156:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 496 00000100:00100000:2.0:1767318768.069174:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 497 00000100:00100000:2.0:1767318768.069181:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982645120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318768.069197:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318768.069307:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982645120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 126us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318768.069315:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 497 00000100:00100000:0.0:1767318768.069359:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982645120:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318768.069766:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318768.069772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318768.069799:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982644992:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318768.580596:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318768.580606:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318768.580677:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318768.580681:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318768.580685:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318768.580693:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4011 00000100:00080000:3.0:1767318768.580698:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318768.580705:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318768.580728:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982645376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318768.580768:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318768.580774:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318768.580776:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318768.580780:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4011 00000100:00080000:3.0:1767318768.580782:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318768.580786:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318768.580797:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982645632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318768.580805:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318768.580808:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318768.580810:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318768.580813:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4011 00000100:00080000:3.0:1767318768.580816:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318768.580819:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318768.580828:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982645760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318768.580848:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318768.580851:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318768.580852:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318768.580855:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4011 00000100:00080000:3.0:1767318768.580857:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318768.580860:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318768.580868:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9f80 x1853163982646144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318768.580904:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982645888:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.580988:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318768.581020:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318768.581026:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982645504:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.581047:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318768.581075:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645504 00000100:00100000:3.0:1767318768.581081:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645504 00000100:00100000:3.0:1767318768.581128:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 713 00000100:00100000:3.0:1767318768.581137:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982645504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.581188:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982645504:12345-0@lo:400:kworker.0 Request processed in 52us (139us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318768.581207:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 713 00010000:00080000:3.0:1767318768.581247:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076faa00 x1853163982645504/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318768.581260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982645504:0@lo:400:kworker.0 00000100:00080000:1.0:1767318768.581354:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318768.581368:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982645248:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.581397:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318768.581424:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318768.581429:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982646016:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.581449:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318768.581473:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645888 00000100:00100000:1.0:1767318768.581477:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645888 00000100:00100000:1.0:1767318768.581499:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646016 00000100:00100000:1.0:1767318768.581502:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646016 00000100:00100000:1.0:1767318768.581520:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 465 00000100:00100000:1.0:1767318768.581526:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982645888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.581557:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982645888:12345-0@lo:400:kworker.0 Request processed in 31us (569us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318768.581562:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 465 00000100:00100000:1.0:1767318768.581575:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 466 00000100:00100000:1.0:1767318768.581580:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982646016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.581605:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982646016:12345-0@lo:400:kworker.0 Request processed in 24us (155us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318768.581610:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 466 00000100:00100000:1.0:1767318768.581636:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982646016:0@lo:400:kworker.0 00000100:00100000:1.0:1767318768.581656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982645888:0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.581713:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645248 00000100:00100000:3.0:1767318768.581717:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645248 00000100:00100000:3.0:1767318768.581789:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000100:00100000:3.0:1767318768.581796:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982645248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318768.581830:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982645248:12345-0@lo:400:kworker.0 Request processed in 35us (433us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318768.581836:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00100000:2.0:1767318768.581907:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982645376:0@lo:38:kworker.0 00000100:00100000:2.0:1767318768.581935:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318768.581961:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982645632:0@lo:38:kworker.0 00000100:00100000:2.0:1767318768.581982:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318768.581999:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982645760:0@lo:38:kworker.0 00000100:00100000:2.0:1767318768.582022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318768.582047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982646144:0@lo:38:kworker.0 00010000:00080000:0.0:1767318768.582058:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9180 x1853163982645248/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318784 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318768.582071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318768.582074:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982645248:0@lo:400:kworker.0 00000100:00100000:0.0:1767318768.582106:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645760 00000100:00100000:2.0:1767318768.582109:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645376 00000100:00100000:0.0:1767318768.582111:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645760 00000100:00100000:2.0:1767318768.582114:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645376 00000100:00100000:0.0:1767318768.582142:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 258 00000100:00100000:2.0:1767318768.582143:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982645632 00000100:00100000:2.0:1767318768.582148:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982645632 00000100:00100000:0.0:1767318768.582150:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982645760:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318768.582163:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646144 00000100:00100000:2.0:1767318768.582166:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646144 00010000:00020000:0.0:1767318768.582171:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318768.582178:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 931 00000100:00100000:2.0:1767318768.582183:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982645376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318768.582206:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318768.582224:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982645760:12345-0@lo:38:kworker.0 Request processed in 74us (200us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318768.582232:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 258 00000100:00100000:2.0:1767318768.582244:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982645376:12345-0@lo:38:kworker.0 Request processed in 57us (305us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318768.582251:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 931 00000100:00100000:0.0:1767318768.582260:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 932 00000100:00100000:0.0:1767318768.582267:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982645632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318768.582309:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318768.582336:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982645632:12345-0@lo:38:kworker.0 Request processed in 69us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318768.582791:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933 00000100:00100000:2.0:1767318768.582800:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982646144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318768.582811:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318768.582842:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982646144:12345-0@lo:38:kworker.0 Request processed in 42us (773us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767318768.583263:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318768.583285:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318768.583289:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982645376:0@lo:38:kworker.0 00000100:00080000:3.0:1767318768.583299:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318768.583302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318768.583305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982645632:0@lo:38:kworker.0 00000100:00080000:3.0:1767318768.583313:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318768.583315:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318768.583318:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982645760:0@lo:38:kworker.0 00000100:00080000:3.0:1767318768.583324:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318768.583327:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318768.583330:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982646144:0@lo:38:kworker.0 00000100:00080000:1.0:1767318768.583463:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318768.584158:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 932 00000100:00100000:2.0:1767318768.586340:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933 00010000:00010000:1.0:1767318768.776049:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318769.800085:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318770.575476:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318770.575528:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318770.575704:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587328 00000100:00100000:2.0:1767318770.575717:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587328 00000100:00100000:2.0:1767318770.575767:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587456 00000100:00100000:2.0:1767318770.575770:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587456 00000100:00100000:2.0:1767318770.575840:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 715 00000100:00100000:2.0:1767318770.575863:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915587456:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318770.575971:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915587456:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 120us (441us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318770.575979:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 715 00000100:00100000:2.0:1767318770.575999:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 934 00000100:00100000:2.0:1767318770.576004:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915587328:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318770.576033:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318770.576064:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915587328:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (616us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318770.576071:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 934 00000100:00080000:1.0:1767318770.576386:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318770.824359:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318771.845575:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318772.868591:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318773.188758:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318773.188797:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982646272:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318773.188914:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318773.189188:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646272 00000100:00100000:3.0:1767318773.189202:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646272 00000100:00100000:3.0:1767318773.189318:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 498 00000100:00100000:3.0:1767318773.189331:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982646272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318773.189375:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318773.189441:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982646272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (528us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318773.189449:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 498 00000100:00100000:3.0:1767318773.189504:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982646272:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318773.189798:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318773.196230:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982646400:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318773.196362:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318773.196498:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646400 00000100:00100000:0.0:1767318773.196509:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646400 00000100:00100000:0.0:1767318773.196581:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 499 00000100:00100000:0.0:1767318773.196592:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982646400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318773.196633:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318773.196693:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982646400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (332us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318773.196702:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 499 00000100:00100000:0.0:1767318773.196765:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982646400:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318773.196847:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318773.700526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318773.700538:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318773.700625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318773.700630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318773.700634:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318773.700645:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4017 00000100:00080000:2.0:1767318773.700652:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318773.700660:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318773.700689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982646656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318773.700746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318773.700750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318773.700752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318773.700756:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4017 00000100:00080000:2.0:1767318773.700759:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318773.700763:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318773.700777:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6d80 x1853163982646912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318773.700785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318773.700788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318773.700789:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318773.700792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4017 00000100:00080000:2.0:1767318773.700794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318773.700797:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318773.700808:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982647040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318773.700829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318773.700832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318773.700834:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318773.700838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4017 00000100:00080000:2.0:1767318773.700841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318773.700844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318773.700855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982647424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318773.700942:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982646656:0@lo:38:kworker.0 00000100:00100000:1.0:1767318773.701017:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318773.701061:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982646912:0@lo:38:kworker.0 00000100:00100000:1.0:1767318773.701095:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318773.701121:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982647040:0@lo:38:kworker.0 00000100:00100000:1.0:1767318773.701143:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318773.701179:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982647424:0@lo:38:kworker.0 00000100:00100000:1.0:1767318773.701197:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318773.701227:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647040 00000100:00100000:1.0:1767318773.701234:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647040 00000100:00100000:1.0:1767318773.701263:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 259 00000100:00100000:1.0:1767318773.701297:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982647040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318773.701316:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318773.701356:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982647040:12345-0@lo:38:kworker.0 Request processed in 85us (211us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318773.701363:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 259 00000100:00100000:1.0:1767318773.701386:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646656 00000100:00100000:1.0:1767318773.701390:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646656 00000100:00100000:1.0:1767318773.701409:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646912 00000100:00100000:1.0:1767318773.701411:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646912 00000100:00100000:1.0:1767318773.701423:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647424 00000100:00100000:1.0:1767318773.701425:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647424 00000100:00100000:1.0:1767318773.701434:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935 00000100:00100000:1.0:1767318773.701438:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982646656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318773.701448:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318773.701475:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982646656:12345-0@lo:38:kworker.0 Request processed in 35us (458us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318773.701481:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935 00000100:00100000:1.0:1767318773.701491:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936 00000100:00100000:1.0:1767318773.701494:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982646912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318773.701501:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318773.701520:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982646912:12345-0@lo:38:kworker.0 Request processed in 25us (425us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318773.701525:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936 00000100:00100000:1.0:1767318773.701533:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 937 00000100:00100000:1.0:1767318773.701536:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982647424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318773.701543:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318773.701563:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982647424:12345-0@lo:38:kworker.0 Request processed in 26us (365us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318773.701570:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 937 00000100:00080000:1.0:1767318773.701603:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318773.701607:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318773.701610:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982646656:0@lo:38:kworker.0 00000100:00080000:1.0:1767318773.701618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318773.701620:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318773.701622:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982646912:0@lo:38:kworker.0 00000100:00080000:1.0:1767318773.701629:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318773.701632:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318773.701634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982647040:0@lo:38:kworker.0 00000100:00080000:1.0:1767318773.701641:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318773.701644:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318773.701647:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982647424:0@lo:38:kworker.0 00000100:00100000:0.0:1767318773.701750:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982647296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.701783:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318773.701815:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318773.701820:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982646528:0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.701840:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318773.701869:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982647168:0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.701892:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318773.701913:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318773.701919:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982646784:0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.701940:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318773.701962:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647296 00000100:00100000:0.0:1767318773.701966:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647296 00000100:00100000:0.0:1767318773.701993:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647168 00000100:00100000:0.0:1767318773.701996:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647168 00000100:00100000:0.0:1767318773.702013:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 467 00000100:00100000:0.0:1767318773.702019:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982647296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.702062:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982647296:12345-0@lo:400:kworker.0 Request processed in 43us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318773.702068:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 467 00000100:00100000:0.0:1767318773.702079:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 468 00000100:00100000:0.0:1767318773.702085:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982647168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.702126:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982647168:12345-0@lo:400:kworker.0 Request processed in 41us (233us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318773.702132:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 468 00000100:00100000:0.0:1767318773.702158:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982647296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318773.702178:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982647168:0@lo:400:kworker.0 00000100:00100000:1.0:1767318773.702251:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646528 00000100:00100000:1.0:1767318773.702255:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646528 00000100:00100000:1.0:1767318773.702302:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982646784 00000100:00100000:1.0:1767318773.702305:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982646784 00000100:00100000:1.0:1767318773.702317:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00100000:1.0:1767318773.702322:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982646528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318773.702353:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982646528:12345-0@lo:400:kworker.0 Request processed in 31us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318773.702358:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00000100:00100000:1.0:1767318773.702368:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 717 00000100:00100000:1.0:1767318773.702372:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982646784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318773.702397:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982646784:12345-0@lo:400:kworker.0 Request processed in 24us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318773.702402:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 717 00010000:00080000:1.0:1767318773.702424:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163982646784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318773.702433:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982646784:0@lo:400:kworker.0 00010000:00080000:1.0:1767318773.702450:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54000 x1853163982646528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318789 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318773.702459:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982646528:0@lo:400:kworker.0 00000100:00080000:0.0:1767318773.702511:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318773.892602:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318774.917297:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318775.695007:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318775.696480:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318775.701909:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587712 00000100:00100000:3.0:1767318775.701936:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587712 00000100:00080000:1.0:1767318775.702082:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318775.702298:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587584 00000100:00100000:0.0:1767318775.702304:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587584 00000100:00100000:0.0:1767318775.702408:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 718 00000100:00100000:0.0:1767318775.702423:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915587712:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318775.702543:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915587712:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 124us (7551us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318775.702552:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 718 00000100:00100000:0.0:1767318775.702571:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 938 00000100:00100000:0.0:1767318775.702577:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915587584:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318775.702604:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318775.702635:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915587584:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (6162us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318775.702642:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 938 00010000:00010000:1.0:1767318775.940703:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318776.964613:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318777.988553:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318778.308924:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982647552:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318778.309558:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318778.309582:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982647680:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318778.310551:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318778.310884:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647680 00000100:00100000:1.0:1767318778.310898:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647680 00000100:00100000:1.0:1767318778.311001:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 500 00000100:00100000:1.0:1767318778.311025:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982647680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318778.311074:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318778.311136:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982647680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (589us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318778.311146:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 500 00000100:00080000:2.0:1767318778.311249:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318778.311328:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982647680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318778.318666:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318778.318834:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647552 00000100:00100000:1.0:1767318778.318845:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647552 00000100:00100000:1.0:1767318778.318916:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 501 00000100:00080000:3.0:1767318778.318918:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318778.318928:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982647552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318778.318965:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318778.319198:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982647552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 273us (542us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318778.319206:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 501 00000100:00100000:3.0:1767318778.321692:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982647552:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318778.823152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318778.823165:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318778.823229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318778.823234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767318778.823237:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767318778.823239:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318778.823250:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4022 00000100:00080000:1.0:1767318778.823257:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318778.823265:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318778.823304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163982647936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318778.823307:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982647808:0@lo:400:kworker.0 00000100:00080000:1.0:1767318778.823351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318778.823355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318778.823356:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318778.823360:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4022 00000100:00080000:1.0:1767318778.823363:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318778.823367:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318778.823382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163982648192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318778.823383:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982647936:0@lo:38:kworker.0 00000100:00080000:1.0:1767318778.823400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767318778.823401:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318778.823406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318778.823409:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318778.823413:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4022 00000100:00080000:1.0:1767318778.823416:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318778.823420:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318778.823433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913800 x1853163982648320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318778.823457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318778.823461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318778.823462:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318778.823466:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4022 00000100:00080000:1.0:1767318778.823469:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767318778.823470:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767318778.823474:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318778.823485:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982648576:0@lo:400:kworker.0 00000100:00100000:1.0:1767318778.823489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982648704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318778.823528:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982648064:0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.823709:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318778.823780:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648576 00000100:00100000:2.0:1767318778.823800:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648576 00000100:00100000:2.0:1767318778.823847:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 469 00000100:00100000:2.0:1767318778.823857:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982648576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.823918:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982648576:12345-0@lo:400:kworker.0 Request processed in 62us (208us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318778.823926:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 469 00000100:00100000:2.0:1767318778.823972:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982648576:0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.824018:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647808 00000100:00100000:2.0:1767318778.824023:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647808 00000100:00100000:2.0:1767318778.824061:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00100000:2.0:1767318778.824068:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982647808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.824102:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982647808:12345-0@lo:400:kworker.0 Request processed in 35us (705us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318778.824108:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00080000:2.0:1767318778.824132:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318778.824160:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982648448:0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.824185:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318778.824213:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648448 00000100:00100000:2.0:1767318778.824217:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648448 00000100:00100000:2.0:1767318778.824241:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 470 00000100:00100000:2.0:1767318778.824247:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982648448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.824302:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982648448:12345-0@lo:400:kworker.0 Request processed in 55us (116us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318778.824307:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 470 00000100:00100000:2.0:1767318778.824331:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982648448:0@lo:400:kworker.0 00010000:00080000:3.0:1767318778.824942:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912a00 x1853163982647808/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318778.824955:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982647808:0@lo:400:kworker.0 00000100:00100000:1.0:1767318778.827627:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318778.828598:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648064 00000100:00100000:2.0:1767318778.828611:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648064 00000100:00100000:2.0:1767318778.828717:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 00000100:00100000:2.0:1767318778.828730:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982648064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318778.828832:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982648064:12345-0@lo:400:kworker.0 Request processed in 106us (1216us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318778.828841:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00010000:00080000:1.0:1767318778.829130:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910e00 x1853163982648064/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318778.829153:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982648064:0@lo:400:kworker.0 00000100:00080000:3.0:1767318778.829478:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318778.833093:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318778.833214:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982648192:0@lo:38:kworker.0 00000100:00100000:0.0:1767318778.834049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318778.834168:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982648320:0@lo:38:kworker.0 00000100:00100000:0.0:1767318778.834212:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318778.834252:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982648704:0@lo:38:kworker.0 00000100:00100000:0.0:1767318778.834299:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318778.834362:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648320 00000100:00100000:3.0:1767318778.834372:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648320 00000100:00100000:3.0:1767318778.834414:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 260 00000100:00100000:3.0:1767318778.834422:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982648320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318778.834443:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318778.834490:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982648320:12345-0@lo:38:kworker.0 Request processed in 68us (276us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318778.834508:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 260 00000100:00080000:0.0:1767318778.838078:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318778.838098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318778.838103:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982648320:0@lo:38:kworker.0 00000100:00100000:2.0:1767318778.841969:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982647936 00000100:00100000:2.0:1767318778.841984:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982647936 00000100:00100000:2.0:1767318778.842056:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648192 00000100:00100000:2.0:1767318778.842060:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648192 00000100:00100000:2.0:1767318778.842082:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648704 00000100:00100000:2.0:1767318778.842085:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648704 00000100:00100000:2.0:1767318778.842103:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939 00000100:00100000:2.0:1767318778.842111:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982647936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318778.842137:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318778.842227:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982647936:12345-0@lo:38:kworker.0 Request processed in 115us (9147us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318778.842234:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939 00000100:00100000:2.0:1767318778.842250:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940 00000100:00100000:2.0:1767318778.842254:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982648192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318778.842263:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318778.842302:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982648192:12345-0@lo:38:kworker.0 Request processed in 46us (8257us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318778.842308:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940 00000100:00100000:2.0:1767318778.842317:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941 00000100:00100000:2.0:1767318778.842321:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982648704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318778.842330:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318778.842362:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982648704:12345-0@lo:38:kworker.0 Request processed in 27us (8050us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318778.842368:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941 00000100:00080000:0.0:1767318778.842687:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318778.842693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318778.842697:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982648192:0@lo:38:kworker.0 00000100:00080000:0.0:1767318778.842707:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318778.842710:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318778.842714:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982648704:0@lo:38:kworker.0 00000100:00080000:0.0:1767318778.842722:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318778.842725:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318778.842728:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982647936:0@lo:38:kworker.0 00010000:00010000:1.0:1767318779.012525:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318780.039042:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318780.813874:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318780.814004:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587968 00000100:00100000:3.0:1767318780.814019:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587968 00000100:00080000:2.0:1767318780.814109:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318780.814121:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 721 00000100:00100000:3.0:1767318780.814134:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915587968:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318780.814233:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915587968:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (371us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318780.814243:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 721 00000100:00100000:1.0:1767318780.819173:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318780.819705:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915587840 00000100:00100000:2.0:1767318780.819729:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915587840 00000100:00100000:2.0:1767318780.819783:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 942 00000100:00100000:2.0:1767318780.819792:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915587840:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318780.819820:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318780.820171:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915587840:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 375us (1009us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318780.820181:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 942 00010000:00010000:1.0:1767318781.060502:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318782.086017:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318783.113768:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318783.428615:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982648832:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318783.428699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318783.428819:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648832 00000100:00100000:2.0:1767318783.428827:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648832 00000100:00100000:2.0:1767318783.428897:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 502 00000100:00100000:2.0:1767318783.428907:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982648832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318783.428953:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318783.429002:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982648832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (302us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318783.429009:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 502 00000100:00080000:3.0:1767318783.429105:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318783.429141:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982648832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318783.429424:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982648960:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318783.429459:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318783.429529:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982648960 00000100:00100000:2.0:1767318783.429533:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982648960 00000100:00100000:2.0:1767318783.429581:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 503 00000100:00100000:2.0:1767318783.429589:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982648960:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318783.429619:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767318783.429667:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318783.429907:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982648960:12345-0@lo:13:osp-pre-0-1.0 Request processed in 317us (445us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318783.429923:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 503 00000100:00100000:1.0:1767318783.429973:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982648960:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318783.940580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318783.940596:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318783.940681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318783.940685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318783.940691:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318783.940700:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4027 00000100:00080000:2.0:1767318783.940708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318783.940717:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318783.940743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4380 x1853163982649216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318783.940957:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318783.940986:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982649088:0@lo:400:kworker.0 00000100:00100000:1.0:1767318783.941080:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318783.941474:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649088 00000100:00100000:0.0:1767318783.941486:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649088 00000100:00100000:0.0:1767318783.941567:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767318783.941581:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982649088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318783.941665:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982649088:12345-0@lo:400:kworker.0 Request processed in 87us (585us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318783.941674:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000100:00080000:3.0:1767318783.941864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318783.941991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318783.941995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318783.941997:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318783.942002:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4027 00000100:00080000:2.0:1767318783.942005:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318783.942009:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318783.942025:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5c00 x1853163982649472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318783.942035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318783.942038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318783.942040:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318783.942043:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4027 00000100:00080000:2.0:1767318783.942045:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318783.942048:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318783.942069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982649600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318783.942092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318783.942095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318783.942097:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318783.942100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4027 00000100:00080000:2.0:1767318783.942103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318783.942106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318783.942116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163982649984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318783.942284:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318783.942298:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982649856:0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942341:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318783.942393:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982649728:0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942415:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318783.942437:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318783.942442:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982649344:0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942460:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318783.942483:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649856 00000100:00100000:3.0:1767318783.942488:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649856 00000100:00100000:3.0:1767318783.942525:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649728 00000100:00100000:3.0:1767318783.942528:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649728 00000100:00100000:3.0:1767318783.942547:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 471 00000100:00100000:3.0:1767318783.942556:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982649856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942594:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982649856:12345-0@lo:400:kworker.0 Request processed in 39us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318783.942601:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 471 00000100:00100000:3.0:1767318783.942612:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 472 00000100:00100000:3.0:1767318783.942617:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982649728:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942654:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982649728:12345-0@lo:400:kworker.0 Request processed in 36us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318783.942659:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 472 00000100:00100000:3.0:1767318783.942691:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982649856:0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942715:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982649728:0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942728:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649344 00000100:00100000:3.0:1767318783.942732:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649344 00000100:00100000:3.0:1767318783.942773:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00100000:3.0:1767318783.942779:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982649344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318783.942809:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982649344:12345-0@lo:400:kworker.0 Request processed in 31us (348us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318783.942814:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00000100:00080000:3.0:1767318783.942827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767318783.942841:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7480 x1853163982649344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318783.942850:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982649344:0@lo:400:kworker.0 00010000:00080000:1.0:1767318783.945548:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982649088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318799 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318783.945571:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982649088:0@lo:400:kworker.0 00000100:00100000:0.0:1767318783.945987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982649216:0@lo:38:kworker.0 00000100:00100000:0.0:1767318783.946245:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318783.946355:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982649472:0@lo:38:kworker.0 00000100:00100000:0.0:1767318783.946381:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318783.946397:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982649600:0@lo:38:kworker.0 00000100:00100000:0.0:1767318783.946422:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318783.946451:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982649984:0@lo:38:kworker.0 00000100:00100000:0.0:1767318783.946476:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318783.946514:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649216 00000100:00100000:0.0:1767318783.946521:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649216 00000100:00100000:0.0:1767318783.946556:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649472 00000100:00100000:0.0:1767318783.946560:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649472 00000100:00100000:0.0:1767318783.946568:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649984 00000100:00100000:0.0:1767318783.946570:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649984 00000100:00100000:0.0:1767318783.946587:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 943 00000100:00100000:0.0:1767318783.946596:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982649216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318783.946622:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318783.946999:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1767318783.947005:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982649472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318783.947017:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318783.947093:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982649472:12345-0@lo:38:kworker.0 Request processed in 85us (709us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318783.947101:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1767318783.947120:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945 00000100:00100000:2.0:1767318783.947124:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982649984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318783.947133:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318783.947160:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982649984:12345-0@lo:38:kworker.0 Request processed in 34us (682us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318783.947166:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945 00000100:00080000:2.0:1767318783.947209:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318783.947215:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318783.947220:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982649472:0@lo:38:kworker.0 00000100:00080000:2.0:1767318783.947230:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318783.947234:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318783.947237:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982649984:0@lo:38:kworker.0 00000100:00100000:2.0:1767318783.947343:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982649600 00000100:00100000:2.0:1767318783.947349:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982649600 00000100:00100000:2.0:1767318783.947379:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 261 00000100:00100000:2.0:1767318783.947384:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982649600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318783.947395:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318783.947427:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982649600:12345-0@lo:38:kworker.0 Request processed in 41us (1003us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318783.947446:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 261 00000100:00080000:2.0:1767318783.947471:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318783.947475:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318783.947479:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982649600:0@lo:38:kworker.0 00000100:00100000:0.0:1767318783.948187:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982649216:12345-0@lo:38:kworker.0 Request processed in 1588us (1941us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318783.948197:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 943 00000100:00080000:2.0:1767318783.948321:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318783.948326:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318783.948330:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982649216:0@lo:38:kworker.0 00010000:00010000:1.0:1767318784.134533:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767318784.908177:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915588096 offset 0 length 520: 4 00000100:00100000:1.0:1767318784.914550:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318784.914588:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318784.914768:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588224 00000100:00100000:0.0:1767318784.914779:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588224 00000100:00100000:0.0:1767318784.914828:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588352 00000100:00100000:0.0:1767318784.914832:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588352 00000100:00100000:0.0:1767318784.914883:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 724 00000100:00100000:0.0:1767318784.914925:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915588352:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318784.915024:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915588352:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (434us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318784.915033:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 724 00000100:00100000:1.0:1767318784.915186:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 946 00000100:00100000:1.0:1767318784.915193:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915588224:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318784.915216:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318784.915246:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915588224:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (706us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318784.915254:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 946 00000100:00080000:2.0:1767318784.915829:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318785.157820:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318786.180939:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318787.204507:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318788.037065:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318763/real 1767318763] req@ffff934639e56a00 x1853163982644096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318788 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318788.037104:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934639e56a00 x1853163982644096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318788 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318788.037124:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318788.037146:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318788.037150:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982644096:0@lo:250:kworker.0 00000100:00080000:3.0:1767318788.037893:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318788.037899:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767318788.037902:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318788.037907:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4011 00000100:00080000:3.0:1767318788.037911:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318788.037917:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318788.037950:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982650112/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318788.038002:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318788.038006:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318788.038008:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318788.038012:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4032 00000100:00080000:3.0:1767318788.038014:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318788.038018:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318788.038034:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163982650368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318788.038059:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318788.038062:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318788.038064:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318788.038068:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4032 00000100:00080000:3.0:1767318788.038070:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318788.038074:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318788.038085:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913800 x1853163982650624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318788.038093:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318788.038096:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318788.038099:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318788.038102:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4032 00000100:00080000:3.0:1767318788.038104:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318788.038108:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318788.038120:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910e00 x1853163982650752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318788.038142:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318788.038145:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318788.038147:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318788.038150:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4032 00000100:00080000:3.0:1767318788.038152:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318788.038155:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318788.038165:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982651136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318788.038192:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982650496:0@lo:400:kworker.0 00000100:00100000:3.0:1767318788.038259:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318788.038336:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318788.038342:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982650880:0@lo:400:kworker.0 00000100:00100000:3.0:1767318788.038368:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.038664:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650496 00000100:00100000:2.0:1767318788.038673:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650496 00000100:00100000:2.0:1767318788.038749:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 725 00000100:00100000:2.0:1767318788.038760:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982650496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318788.038821:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982650496:12345-0@lo:400:kworker.0 Request processed in 63us (561us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318788.038838:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 725 00000100:00100000:2.0:1767318788.038966:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650880 00000100:00100000:2.0:1767318788.038971:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650880 00000100:00100000:2.0:1767318788.038998:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 473 00000100:00100000:2.0:1767318788.039006:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982650880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.039019:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982651008:0@lo:400:kworker.0 00000100:00100000:2.0:1767318788.039042:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982650880:12345-0@lo:400:kworker.0 Request processed in 37us (672us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318788.039049:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 473 00000100:00100000:1.0:1767318788.039060:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.039072:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982650112:0@lo:250:kworker.0 00000100:00100000:1.0:1767318788.039092:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000400:00000100:2.0:1767318788.039101:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982650112 offset 0 length 520: 4 00000100:00100000:1.0:1767318788.039102:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982650240:0@lo:400:kworker.0 00000100:00100000:2.0:1767318788.039125:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982650368:0@lo:38:kworker.0 00000100:00100000:1.0:1767318788.039142:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.039161:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650240 00000100:00100000:2.0:1767318788.039166:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650240 00000100:00100000:1.0:1767318788.039187:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651008 00000100:00100000:1.0:1767318788.039193:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651008 00000100:00100000:2.0:1767318788.039198:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00100000:2.0:1767318788.039208:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982650240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.039229:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 474 00000100:00100000:1.0:1767318788.039238:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982651008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318788.039248:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982650240:12345-0@lo:400:kworker.0 Request processed in 42us (106us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318788.039255:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00000100:00100000:1.0:1767318788.039311:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982651008:12345-0@lo:400:kworker.0 Request processed in 75us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318788.039319:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 474 00010000:00080000:2.0:1767318788.039322:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fad80 x1853163982650240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318804 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318788.039337:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982650240:0@lo:400:kworker.0 00000100:00100000:2.0:1767318788.039351:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982651008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.039361:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318788.039386:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982650624:0@lo:38:kworker.0 00000100:00100000:2.0:1767318788.039402:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982650880:0@lo:400:kworker.0 00010000:00080000:2.0:1767318788.039417:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910380 x1853163982650496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318804 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318788.039424:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.039435:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982650496:0@lo:400:kworker.0 00000100:00100000:1.0:1767318788.039451:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982650752:0@lo:38:kworker.0 00000100:00100000:2.0:1767318788.039459:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650368 00000100:00100000:2.0:1767318788.039464:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650368 00000100:00100000:1.0:1767318788.039481:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.039484:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650624 00000100:00100000:2.0:1767318788.039488:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650624 00000100:00100000:1.0:1767318788.039501:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982651136:0@lo:38:kworker.0 00000100:00100000:2.0:1767318788.039508:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947 00000100:00100000:2.0:1767318788.039513:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982650368:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318788.039525:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318788.039548:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651136 00000100:00100000:1.0:1767318788.039552:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651136 00000100:00100000:1.0:1767318788.039566:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948 00000100:00100000:1.0:1767318788.039571:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982650624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318788.039590:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318788.039644:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318788.039649:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318788.039652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318788.039666:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982650752 00000100:00100000:3.0:1767318788.039671:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982650752 00000100:00100000:3.0:1767318788.039698:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 262 00000100:00100000:3.0:1767318788.039704:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982650752:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318788.039714:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318788.039753:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982650752:12345-0@lo:38:kworker.0 Request processed in 49us (273us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318788.039759:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 262 00000100:00100000:3.0:1767318788.039789:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 949 00000100:00100000:3.0:1767318788.039793:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651136:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318788.039808:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318788.039844:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651136:12345-0@lo:38:kworker.0 Request processed in 49us (319us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767318788.039940:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318788.039946:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318788.039950:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982650752:0@lo:38:kworker.0 00000100:00080000:0.0:1767318788.039959:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318788.039962:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318788.039965:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982651136:0@lo:38:kworker.0 00000100:00100000:3.0:1767318788.039988:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 949 00010000:00020000:2.0:1767318788.040052:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318788.040139:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982650368:12345-0@lo:38:kworker.0 Request processed in 623us (779us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318788.040146:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947 00000100:00080000:0.0:1767318788.040195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318788.040200:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318788.040203:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982650368:0@lo:38:kworker.0 00000100:00100000:1.0:1767318788.068066:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982650624:12345-0@lo:38:kworker.0 Request processed in 28479us (28630us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318788.068105:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948 00000100:00080000:0.0:1767318788.068364:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318788.068374:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318788.068379:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982650624:0@lo:38:kworker.0 00010000:00010000:1.0:1767318788.228566:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318788.548940:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318788.548983:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982651264:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318788.549056:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318788.549493:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651264 00000100:00100000:0.0:1767318788.549504:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651264 00000100:00100000:0.0:1767318788.549564:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 504 00000100:00100000:0.0:1767318788.549574:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982651264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318788.549620:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318788.549680:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982651264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (623us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318788.549688:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 504 00000100:00100000:2.0:1767318788.549785:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982651264:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318788.551365:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982651392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318788.551429:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318788.552112:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651392 00000100:00100000:2.0:1767318788.552121:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651392 00000100:00100000:2.0:1767318788.552186:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 505 00000100:00100000:2.0:1767318788.552197:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982651392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318788.552663:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318788.552731:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982651392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 535us (1300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318788.552743:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 505 00000100:00100000:3.0:1767318788.552866:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982651392:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318789.253742:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318790.027887:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318790.028012:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588480 00000100:00100000:1.0:1767318790.028026:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588480 00000100:00100000:1.0:1767318790.028100:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 950 00000100:00100000:1.0:1767318790.028110:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915588480:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318790.028140:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318790.028254:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915588480:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 142us (380us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318790.028263:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 950 00000100:00100000:3.0:1767318790.033475:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318790.034453:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588608 00000100:00100000:2.0:1767318790.034480:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588608 00000100:00100000:2.0:1767318790.034828:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 727 00000100:00100000:2.0:1767318790.034841:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915588608:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318790.035497:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915588608:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 654us (2037us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318790.035510:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 727 00000100:00080000:3.0:1767318790.038500:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318790.277450:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318791.300718:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318792.324935:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318793.157320:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318793.157332:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318793.157398:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318793.157402:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318793.157406:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318793.157415:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4036 00000100:00080000:2.0:1767318793.157421:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318793.157428:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318793.157455:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5180 x1853163982651648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318793.157474:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982651520:0@lo:400:kworker.0 00000100:00080000:2.0:1767318793.157500:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318793.157505:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318793.157518:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318793.157523:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4036 00000100:00080000:2.0:1767318793.157525:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318793.157531:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318793.157544:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982651904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318793.157552:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318793.157560:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318793.157564:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318793.157566:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318793.157570:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4036 00000100:00080000:2.0:1767318793.157572:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318793.157576:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318793.157588:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982652032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318793.157610:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318793.157612:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318793.157614:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318793.157617:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4036 00000100:00080000:2.0:1767318793.157619:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318793.157623:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318793.157632:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4000 x1853163982652416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318793.157657:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982651776:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.157660:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318793.157670:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982652288:0@lo:400:kworker.0 00000100:00100000:2.0:1767318793.157693:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.157771:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982651648:0@lo:38:kworker.0 00000100:00100000:0.0:1767318793.158371:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651520 00000100:00100000:0.0:1767318793.158380:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651520 00000100:00100000:0.0:1767318793.158442:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651776 00000100:00100000:0.0:1767318793.158446:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651776 00000100:00100000:0.0:1767318793.158473:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 728 00000100:00100000:0.0:1767318793.158482:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982651520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.161242:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318793.161740:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318793.161778:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982652160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.161861:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318793.161920:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 729 00000100:00100000:3.0:1767318793.161935:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982651776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.161987:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982651776:12345-0@lo:400:kworker.0 Request processed in 59us (4295us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318793.161996:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 729 00000100:00100000:3.0:1767318793.162061:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652288 00000100:00100000:3.0:1767318793.162068:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652288 00000100:00100000:3.0:1767318793.162111:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652160 00000100:00100000:3.0:1767318793.162114:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652160 00000100:00100000:3.0:1767318793.162765:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 475 00000100:00100000:3.0:1767318793.162773:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982652288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.162852:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982652288:12345-0@lo:400:kworker.0 Request processed in 77us (1615us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318793.162862:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 475 00000100:00100000:3.0:1767318793.162887:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 476 00000100:00100000:3.0:1767318793.162898:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982652160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.162948:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982652160:12345-0@lo:400:kworker.0 Request processed in 54us (1087us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318793.162953:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 476 00000100:00080000:3.0:1767318793.162970:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318793.162974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318793.163009:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982652160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318793.163055:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982652288:0@lo:400:kworker.0 00010000:00080000:2.0:1767318793.167432:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7b80 x1853163982651776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318809 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318793.167451:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982651776:0@lo:400:kworker.0 00000100:00100000:1.0:1767318793.170285:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.170355:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982651904:0@lo:38:kworker.0 00000100:00100000:1.0:1767318793.170403:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.170441:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651648 00000100:00100000:1.0:1767318793.170449:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651648 00000100:00100000:1.0:1767318793.170485:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982651904 00000100:00100000:1.0:1767318793.170488:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982651904 00000100:00100000:1.0:1767318793.170514:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951 00000100:00100000:1.0:1767318793.170521:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318793.170542:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318793.170589:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651648:12345-0@lo:38:kworker.0 Request processed in 67us (325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318793.170596:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951 00000100:00100000:1.0:1767318793.170614:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952 00000100:00100000:1.0:1767318793.170619:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318793.170628:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318793.170651:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982651904:12345-0@lo:38:kworker.0 Request processed in 31us (248us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318793.170657:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952 00000100:00080000:1.0:1767318793.170708:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318793.170714:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318793.170717:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982651904:0@lo:38:kworker.0 00000100:00100000:1.0:1767318793.170744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982652032:0@lo:38:kworker.0 00000100:00100000:1.0:1767318793.170774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.170808:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982652416:0@lo:38:kworker.0 00000100:00100000:1.0:1767318793.170829:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318793.170866:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318793.170871:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318793.170874:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982651648:0@lo:38:kworker.0 00000100:00100000:3.0:1767318793.171245:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652032 00000100:00100000:3.0:1767318793.171251:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652032 00000100:00100000:3.0:1767318793.171295:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 263 00000100:00100000:3.0:1767318793.171302:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982652032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318793.171326:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318793.171364:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982652032:12345-0@lo:38:kworker.0 Request processed in 62us (589us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318793.171372:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 263 00000100:00100000:3.0:1767318793.171389:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652416 00000100:00100000:3.0:1767318793.171392:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652416 00000100:00100000:3.0:1767318793.171409:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953 00000100:00100000:3.0:1767318793.171414:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982652416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318793.171423:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318793.171445:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982652416:12345-0@lo:38:kworker.0 Request processed in 30us (615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318793.171450:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953 00000100:00080000:1.0:1767318793.173365:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318793.173381:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318793.173387:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982652032:0@lo:38:kworker.0 00000100:00080000:1.0:1767318793.173403:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318793.173406:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318793.173409:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982652416:0@lo:38:kworker.0 00000100:00100000:0.0:1767318793.177501:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982651520:12345-0@lo:400:kworker.0 Request processed in 19006us (19939us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318793.177521:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 728 00010000:00080000:3.0:1767318793.177698:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5c00 x1853163982651520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318809 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318793.177728:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982651520:0@lo:400:kworker.0 00010000:00010000:1.0:1767318793.349096:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318793.669303:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318793.669329:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982652544:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318793.669457:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318793.669622:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652544 00000100:00100000:2.0:1767318793.669631:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652544 00000100:00100000:2.0:1767318793.669702:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 506 00000100:00100000:2.0:1767318793.669713:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982652544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318793.669759:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318793.669831:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982652544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (372us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318793.669841:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 506 00000100:00100000:0.0:1767318793.670078:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982652544:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318793.670204:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982652672:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318793.670234:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318793.670288:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652672 00000100:00100000:1.0:1767318793.670293:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652672 00000100:00100000:1.0:1767318793.670338:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 507 00000100:00100000:1.0:1767318793.670345:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982652672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318793.670361:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318793.670956:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982652672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 534us (644us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318793.670966:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 507 00000100:00100000:0.0:1767318793.671029:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982652672:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318794.372766:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318795.155813:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318795.155870:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318795.156191:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588736 00000100:00100000:3.0:1767318795.156201:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588736 00000100:00100000:3.0:1767318795.156249:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588864 00000100:00100000:3.0:1767318795.156252:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588864 00000100:00100000:1.0:1767318795.156344:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 730 00000100:00100000:1.0:1767318795.156357:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915588864:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767318795.156383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318795.156436:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915588864:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (565us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318795.163898:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 954 00000100:00100000:3.0:1767318795.163914:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915588736:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318795.163940:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318795.164646:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915588736:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 731us (8841us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318795.164657:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 954 00000100:00100000:1.0:1767318795.168255:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 730 00010000:00010000:1.0:1767318795.396867:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318796.420544:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318797.446459:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318798.277698:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.277708:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318798.277772:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.277777:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.277781:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.277789:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.277794:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.277815:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318798.277853:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961500 x1853163982652928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.277884:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.277887:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.277890:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.277895:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.277897:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.277901:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318798.277913:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163982653184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.277922:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.277925:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.277927:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.277930:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.277932:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.277936:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318798.277947:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982653312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318798.277969:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318798.277973:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318798.277975:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318798.277978:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4041 00000100:00080000:3.0:1767318798.277980:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318798.277984:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318798.277997:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91f80 x1853163982653696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318798.278090:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982653568:0@lo:400:kworker.0 00000100:00100000:2.0:1767318798.278157:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318798.278205:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318798.278212:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982652800:0@lo:400:kworker.0 00000100:00100000:2.0:1767318798.278237:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318798.278286:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652800 00000100:00100000:2.0:1767318798.278293:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652800 00000100:00100000:2.0:1767318798.278362:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00100000:2.0:1767318798.278371:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982652800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318798.278421:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982652800:12345-0@lo:400:kworker.0 Request processed in 52us (183us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318798.278429:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00000100:00080000:2.0:1767318798.278451:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318798.278478:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962a00 x1853163982652800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318814 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318798.278491:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982652800:0@lo:400:kworker.0 00000100:00100000:3.0:1767318798.278659:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653568 00000100:00100000:3.0:1767318798.278663:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653568 00000100:00100000:3.0:1767318798.278700:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 477 00000100:00100000:3.0:1767318798.278707:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982653568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318798.278743:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982653568:12345-0@lo:400:kworker.0 Request processed in 37us (586us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318798.278749:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 477 00000100:00080000:2.0:1767318798.278809:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318798.278836:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982653568:0@lo:400:kworker.0 00000100:00100000:1.0:1767318798.279931:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982652928:0@lo:38:kworker.0 00000100:00100000:1.0:1767318798.279998:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318798.280027:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982653184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318798.280048:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318798.280077:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982653312:0@lo:38:kworker.0 00000100:00100000:1.0:1767318798.280127:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318798.280169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982653696:0@lo:38:kworker.0 00000100:00100000:1.0:1767318798.280190:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318798.280255:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653312 00000100:00100000:1.0:1767318798.280261:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653312 00000100:00100000:1.0:1767318798.280321:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 264 00000100:00100000:1.0:1767318798.280328:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982653312:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318798.280347:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318798.280385:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982653312:12345-0@lo:38:kworker.0 Request processed in 56us (257us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318798.280392:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 264 00000100:00100000:1.0:1767318798.280413:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982652928 00000100:00100000:1.0:1767318798.280430:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982652928 00000100:00100000:1.0:1767318798.280448:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653184 00000100:00100000:1.0:1767318798.280450:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653184 00000100:00100000:1.0:1767318798.280460:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653696 00000100:00100000:1.0:1767318798.280462:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653696 00000100:00100000:1.0:1767318798.280473:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 955 00000100:00100000:1.0:1767318798.280479:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982652928:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318798.280489:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318798.280501:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956 00000100:00100000:3.0:1767318798.280509:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982653184:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318798.280517:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982652928:12345-0@lo:38:kworker.0 Request processed in 36us (518us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767318798.280523:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318798.280527:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 955 00000100:00100000:1.0:1767318798.280539:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957 00000100:00100000:1.0:1767318798.280544:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982653696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318798.280554:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318798.280565:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982653184:12345-0@lo:38:kworker.0 Request processed in 54us (515us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318798.280573:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956 00000100:00100000:1.0:1767318798.280587:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982653696:12345-0@lo:38:kworker.0 Request processed in 41us (394us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318798.280594:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957 00000100:00080000:1.0:1767318798.280648:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318798.280653:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318798.280656:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982652928:0@lo:38:kworker.0 00000100:00080000:1.0:1767318798.280665:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318798.280669:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318798.280671:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982653184:0@lo:38:kworker.0 00000100:00080000:1.0:1767318798.280678:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318798.280681:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318798.280683:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982653312:0@lo:38:kworker.0 00000100:00080000:1.0:1767318798.280691:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318798.280694:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318798.280696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982653696:0@lo:38:kworker.0 00000100:00100000:0.0:1767318798.280856:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982653440:0@lo:400:kworker.0 00000100:00100000:0.0:1767318798.280892:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318798.280922:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318798.280929:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982653056:0@lo:400:kworker.0 00000100:00100000:0.0:1767318798.280951:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318798.281038:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653440 00000100:00100000:3.0:1767318798.281042:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653440 00000100:00100000:3.0:1767318798.281102:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 478 00000100:00100000:3.0:1767318798.281110:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982653440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318798.281158:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982653440:12345-0@lo:400:kworker.0 Request processed in 49us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318798.281170:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 478 00000100:00100000:2.0:1767318798.281217:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653056 00000100:00100000:2.0:1767318798.281222:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653056 00000100:00100000:2.0:1767318798.281302:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000100:00100000:2.0:1767318798.281312:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982653056:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318798.281319:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318798.281326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318798.281353:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982653440:0@lo:400:kworker.0 00000100:00100000:2.0:1767318798.281379:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982653056:12345-0@lo:400:kworker.0 Request processed in 69us (426us total) trans 0 rc 0/0 00010000:00080000:0.0:1767318798.281383:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963100 x1853163982653056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318814 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318798.281390:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00100000:0.0:1767318798.281403:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982653056:0@lo:400:kworker.0 00010000:00010000:1.0:1767318798.468590:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318798.789121:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982653824:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318798.789193:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318798.789358:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653824 00000100:00100000:0.0:1767318798.789367:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653824 00000100:00100000:0.0:1767318798.789426:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 508 00000100:00100000:0.0:1767318798.789433:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982653824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318798.789460:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318798.789505:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982653824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318798.789513:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 508 00000100:00100000:1.0:1767318798.789521:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982653952:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318798.789558:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318798.789569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982653824:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318798.789598:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982653952 00000100:00100000:1.0:1767318798.789603:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982653952 00000100:00100000:1.0:1767318798.789662:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 509 00000100:00100000:1.0:1767318798.789669:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982653952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318798.789687:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318798.789717:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982653952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (159us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318798.789724:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 509 00000100:00100000:1.0:1767318798.789754:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982653952:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318799.492716:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318800.273478:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318800.273536:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318800.273953:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589120 00000100:00100000:3.0:1767318800.273963:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589120 00000100:00100000:3.0:1767318800.274052:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915588992 00000100:00100000:3.0:1767318800.274056:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915588992 00000100:00100000:3.0:1767318800.274085:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 733 00000100:00100000:3.0:1767318800.274095:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915589120:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318800.274189:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915589120:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (733us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318800.274197:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 733 00000100:00100000:3.0:1767318800.274216:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 958 00000100:00100000:3.0:1767318800.274221:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915588992:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318800.274239:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318800.274264:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915588992:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (728us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318800.274316:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 958 00000100:00080000:0.0:1767318800.275140:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318800.518976:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318801.540537:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318802.564915:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318803.408515:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318803.408526:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318803.408631:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767318803.409393:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318803.409399:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318803.409403:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318803.409410:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4046 00000100:00080000:0.0:1767318803.409415:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318803.409422:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318803.409488:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9880 x1853163982654208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318803.409554:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318803.409558:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318803.409560:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318803.409564:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4046 00000100:00080000:0.0:1767318803.409566:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318803.409570:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318803.409580:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab100 x1853163982654464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318803.409588:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318803.409591:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318803.409593:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318803.409596:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4046 00000100:00080000:0.0:1767318803.409598:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318803.409602:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318803.409625:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab480 x1853163982654592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318803.409696:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982654208:0@lo:38:kworker.0 00000100:00100000:2.0:1767318803.409756:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.409818:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982654464:0@lo:38:kworker.0 00000100:00100000:2.0:1767318803.409848:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.409864:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982654592:0@lo:38:kworker.0 00000100:00100000:2.0:1767318803.409893:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.409938:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654208 00000100:00100000:2.0:1767318803.409944:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654208 00000100:00100000:2.0:1767318803.409968:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654464 00000100:00100000:2.0:1767318803.409971:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654464 00000100:00100000:2.0:1767318803.409986:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959 00000100:00100000:2.0:1767318803.409993:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982654208:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318803.410025:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318803.410059:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982654208:12345-0@lo:38:kworker.0 Request processed in 66us (302us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318803.410066:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959 00000100:00100000:2.0:1767318803.410080:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960 00000100:00100000:2.0:1767318803.410084:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982654464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318803.410092:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318803.410123:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982654464:12345-0@lo:38:kworker.0 Request processed in 38us (275us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318803.410129:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960 00000100:00080000:2.0:1767318803.410174:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318803.410178:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318803.410181:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982654208:0@lo:38:kworker.0 00000100:00080000:2.0:1767318803.410189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318803.410192:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318803.410195:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982654464:0@lo:38:kworker.0 00000100:00100000:2.0:1767318803.410234:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982654720:0@lo:400:kworker.0 00000100:00100000:2.0:1767318803.410304:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.410321:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318803.410326:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982654336:0@lo:400:kworker.0 00000100:00100000:2.0:1767318803.410345:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.410367:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654336 00000100:00100000:2.0:1767318803.410370:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654336 00000100:00100000:2.0:1767318803.410403:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00100000:2.0:1767318803.410409:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982654336:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318803.410442:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982654336:12345-0@lo:400:kworker.0 Request processed in 33us (96us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318803.410447:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00010000:00080000:2.0:1767318803.410470:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9f80 x1853163982654336/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318819 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318803.410480:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982654336:0@lo:400:kworker.0 00000100:00080000:2.0:1767318803.410500:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318803.410657:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654592 00000100:00100000:3.0:1767318803.410675:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654592 00000100:00100000:3.0:1767318803.410705:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 265 00000100:00100000:3.0:1767318803.410710:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982654592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318803.410720:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318803.410752:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982654592:12345-0@lo:38:kworker.0 Request processed in 42us (859us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318803.410758:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 265 00000100:00100000:3.0:1767318803.410773:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654720 00000100:00100000:3.0:1767318803.410776:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654720 00000100:00100000:3.0:1767318803.410801:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 479 00000100:00100000:3.0:1767318803.410806:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982654720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318803.410834:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982654720:12345-0@lo:400:kworker.0 Request processed in 28us (539us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318803.410840:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 479 00000100:00080000:3.0:1767318803.410854:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318803.410868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982654720:0@lo:400:kworker.0 00000100:00080000:3.0:1767318803.410887:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318803.410890:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318803.410894:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982654592:0@lo:38:kworker.0 00000100:00100000:1.0:1767318803.424440:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982654848:0@lo:400:kworker.0 00000100:00100000:1.0:1767318803.424578:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318803.424626:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982654080:0@lo:400:kworker.0 00000100:00100000:1.0:1767318803.424651:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318803.424768:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654080 00000100:00100000:2.0:1767318803.424777:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654080 00000100:00100000:2.0:1767318803.424838:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00100000:2.0:1767318803.424847:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982654080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318803.424895:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982654080:12345-0@lo:400:kworker.0 Request processed in 49us (243us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318803.424902:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00100000:3.0:1767318803.424987:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654848 00000100:00100000:3.0:1767318803.424991:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654848 00000100:00100000:3.0:1767318803.425032:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 480 00000100:00100000:3.0:1767318803.425039:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982654848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318803.425072:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982654848:12345-0@lo:400:kworker.0 Request processed in 34us (495us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318803.425077:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 480 00000100:00080000:1.0:1767318803.425464:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318803.425468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318803.425498:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982654848:0@lo:400:kworker.0 00010000:00080000:1.0:1767318803.425509:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aaa00 x1853163982654080/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318819 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318803.425519:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982654080:0@lo:400:kworker.0 00000100:00080000:0.0:1767318803.428213:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318803.428222:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318803.428226:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318803.428242:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4046 00000100:00080000:0.0:1767318803.428247:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318803.428255:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318803.428350:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa300 x1853163982654976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318803.428513:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982654976:0@lo:38:kworker.0 00000100:00100000:2.0:1767318803.428583:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318803.428818:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982654976 00000100:00100000:0.0:1767318803.428825:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982654976 00000100:00100000:0.0:1767318803.428863:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 961 00000100:00100000:0.0:1767318803.428871:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982654976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318803.428891:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318803.428939:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982654976:12345-0@lo:38:kworker.0 Request processed in 68us (356us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318803.428946:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 961 00000100:00080000:3.0:1767318803.429111:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318803.429117:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318803.429120:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982654976:0@lo:38:kworker.0 00010000:00010000:1.0:1767318803.588651:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318803.908600:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982655104:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318803.908665:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318803.909161:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982655232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318803.909190:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318803.909311:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655104 00000100:00100000:3.0:1767318803.909318:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655104 00000100:00100000:3.0:1767318803.909365:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655232 00000100:00100000:3.0:1767318803.909368:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655232 00000100:00100000:3.0:1767318803.909383:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 510 00000100:00100000:3.0:1767318803.909391:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982655104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318803.909420:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318803.909465:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982655104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (800us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318803.909472:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 510 00000100:00100000:3.0:1767318803.909488:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 511 00000100:00100000:3.0:1767318803.909494:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982655232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318803.909509:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318803.909547:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982655232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318803.909554:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 511 00000100:00100000:3.0:1767318803.909590:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982655104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318803.909764:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982655232:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318804.612715:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318805.388839:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318805.388884:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318805.388997:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589248 00000100:00100000:1.0:1767318805.389010:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589248 00000100:00100000:1.0:1767318805.389061:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589376 00000100:00100000:1.0:1767318805.389065:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589376 00000100:00100000:1.0:1767318805.389124:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 736 00000100:00100000:1.0:1767318805.389134:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915589376:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318805.389206:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915589376:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 73us (320us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318805.389213:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 736 00000100:00100000:1.0:1767318805.389247:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 962 00000100:00100000:1.0:1767318805.389251:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915589248:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318805.389308:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318805.389335:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915589248:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 81us (508us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318805.389341:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 962 00000100:00080000:0.0:1767318805.389606:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318805.636528:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318806.660526:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318807.684695:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318808.517308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318808.517320:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318808.517415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318808.517419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318808.517424:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318808.517434:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4051 00000100:00080000:2.0:1767318808.517440:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318808.517456:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318808.517489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163982655488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318808.517551:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318808.517555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318808.517557:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318808.517562:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4051 00000100:00080000:2.0:1767318808.517565:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318808.517570:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318808.517585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982655744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318808.517596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318808.517600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318808.517602:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767318808.517604:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982655488:0@lo:38:kworker.0 00000100:00080000:2.0:1767318808.517606:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4051 00000100:00080000:2.0:1767318808.517612:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318808.517618:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318808.517633:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982655872/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318808.517658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318808.517662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318808.517664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318808.517668:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4051 00000100:00080000:2.0:1767318808.517670:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318808.517674:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318808.517686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960e00 x1853163982656256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318808.517694:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318808.517727:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982655616:0@lo:400:kworker.0 00000100:00100000:2.0:1767318808.517767:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318808.517791:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982655744:0@lo:38:kworker.0 00000100:00100000:0.0:1767318808.517867:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982656128:0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.517903:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318808.517929:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318808.517935:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982655360:0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.517972:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656128 00000100:00100000:0.0:1767318808.517980:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656128 00000100:00100000:0.0:1767318808.518031:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 481 00000100:00100000:0.0:1767318808.518040:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982656128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.518088:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982656128:12345-0@lo:400:kworker.0 Request processed in 50us (184us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318808.518095:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 481 00000100:00080000:0.0:1767318808.518117:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318808.518144:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982656000:0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.518167:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318808.518195:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656000 00000100:00100000:0.0:1767318808.518199:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656000 00000100:00100000:0.0:1767318808.518226:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 482 00000100:00100000:0.0:1767318808.518232:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982656000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.518265:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982656000:12345-0@lo:400:kworker.0 Request processed in 32us (97us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318808.518290:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 482 00000100:00080000:0.0:1767318808.518303:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318808.518331:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982656000:0@lo:400:kworker.0 00000100:00100000:2.0:1767318808.518509:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318808.525086:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655488 00000100:00100000:1.0:1767318808.525102:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655488 00000100:00100000:2.0:1767318808.525126:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982656128:0@lo:400:kworker.0 00000100:00100000:1.0:1767318808.525149:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655616 00000100:00100000:1.0:1767318808.525154:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655616 00000100:00100000:2.0:1767318808.525186:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318808.525244:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655360 00000100:00100000:3.0:1767318808.525249:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655360 00000100:00100000:3.0:1767318808.525284:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655744 00000100:00100000:3.0:1767318808.525287:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655744 00000100:00100000:3.0:1767318808.525299:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 737 00000100:00100000:3.0:1767318808.525309:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982655616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318808.525581:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00100000:1.0:1767318808.525670:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963 00000100:00100000:1.0:1767318808.525676:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982655488:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318808.525695:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318808.525754:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318808.525767:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982655360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318808.527047:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982655616:12345-0@lo:400:kworker.0 Request processed in 1733us (9273us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318808.527061:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 737 00000100:00100000:3.0:1767318808.527097:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 964 00000100:00100000:3.0:1767318808.527102:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982655744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318808.527119:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318808.527155:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982655744:12345-0@lo:38:kworker.0 Request processed in 51us (1969us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318808.527160:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 964 00010000:00080000:3.0:1767318808.527238:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963100 x1853163982655616/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318824 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318808.527251:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982655616:0@lo:400:kworker.0 00000100:00100000:1.0:1767318808.531043:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982655488:12345-0@lo:38:kworker.0 Request processed in 5353us (13341us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318808.531060:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963 00000100:00100000:0.0:1767318808.531236:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982655360:12345-0@lo:400:kworker.0 Request processed in 5645us (12724us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318808.531246:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00010000:00080000:3.0:1767318808.531546:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6d80 x1853163982655360/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318824 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318808.531563:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982655360:0@lo:400:kworker.0 00000100:00080000:2.0:1767318808.536988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318808.537011:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318808.537018:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982655744:0@lo:38:kworker.0 00000100:00100000:2.0:1767318808.537036:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982655872:0@lo:38:kworker.0 00000100:00100000:2.0:1767318808.537131:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318808.537180:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982656256:0@lo:38:kworker.0 00000100:00100000:1.0:1767318808.537194:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982655872 00000100:00100000:1.0:1767318808.537206:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982655872 00000100:00100000:2.0:1767318808.537252:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318808.537268:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 266 00000100:00100000:1.0:1767318808.537302:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982655872:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767318808.537315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318808.537321:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318808.537326:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982655488:0@lo:38:kworker.0 00010000:00020000:1.0:1767318808.537329:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318808.537406:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982655872:12345-0@lo:38:kworker.0 Request processed in 103us (274us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318808.537413:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 266 00000100:00100000:0.0:1767318808.537817:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656256 00000100:00100000:0.0:1767318808.537823:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656256 00000100:00100000:0.0:1767318808.537859:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965 00000100:00100000:0.0:1767318808.537865:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982656256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318808.537877:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318808.537912:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982656256:12345-0@lo:38:kworker.0 Request processed in 46us (661us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318808.537918:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965 00000100:00080000:2.0:1767318808.540607:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318808.540623:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318808.540629:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982655872:0@lo:38:kworker.0 00000100:00080000:2.0:1767318808.540643:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318808.540646:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318808.540649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982656256:0@lo:38:kworker.0 00010000:00010000:1.0:1767318808.708546:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318809.029724:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318809.029864:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318809.029885:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982656512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318809.029952:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318809.030032:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656512 00000100:00100000:0.0:1767318809.030040:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656512 00000100:00100000:0.0:1767318809.030092:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 512 00000100:00100000:0.0:1767318809.030102:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982656512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318809.030135:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318809.030181:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982656512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (228us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318809.030189:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 512 00000100:00100000:0.0:1767318809.030236:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982656512:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318809.037998:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982656384:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318809.038140:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318809.053863:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656384 00000100:00100000:0.0:1767318809.053880:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656384 00000100:00100000:0.0:1767318809.053952:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 513 00000100:00100000:0.0:1767318809.053964:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982656384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318809.054007:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318809.054162:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982656384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 196us (16021us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318809.054176:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 513 00000100:00100000:3.0:1767318809.056824:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982656384:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318809.736355:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318810.509778:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318810.509887:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767318810.512770:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915589760 offset 0 length 520: 4 00000100:00100000:2.0:1767318810.516138:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589632 00000100:00100000:2.0:1767318810.516521:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589632 00000100:00100000:1.0:1767318810.517325:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589504 00000100:00100000:1.0:1767318810.517335:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589504 00000100:00100000:1.0:1767318810.517438:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 966 00000100:00100000:1.0:1767318810.517449:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915589504:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318810.517484:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318810.517600:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915589504:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 150us (7710us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318810.517609:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 966 00000100:00100000:3.0:1767318810.522785:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 739 00000100:00100000:3.0:1767318810.522803:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915589632:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318810.522949:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915589632:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 149us (13184us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318810.522957:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 739 00000100:00080000:0.0:1767318810.531452:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318810.760240:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318811.780941:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318812.804682:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318813.638929:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318788/real 1767318788] req@ffff9346076f9c00 x1853163982650112/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318813 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318813.638986:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076f9c00 x1853163982650112/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318813 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318813.639015:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318813.639052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318813.639056:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982650112:0@lo:250:kworker.0 00000100:00080000:1.0:1767318813.639137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318813.639146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318813.639150:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318813.639156:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4036 00000100:00080000:1.0:1767318813.639161:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318813.639170:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318813.639219:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982656640/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318813.639312:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982656640:0@lo:250:kworker.0 00000400:00000100:0.0:1767318813.639409:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982656640 offset 0 length 520: 4 00000100:00080000:1.0:1767318813.639593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318813.639598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318813.639600:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318813.639606:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4057 00000100:00080000:1.0:1767318813.639609:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318813.639614:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318813.639632:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1f80 x1853163982656896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318813.639646:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982656768:0@lo:400:kworker.0 00000100:00100000:2.0:1767318813.639704:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.639755:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656768 00000100:00100000:3.0:1767318813.639767:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656768 00000100:00080000:2.0:1767318813.639846:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318813.639864:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 740 00000100:00100000:3.0:1767318813.639880:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982656768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318813.639967:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982656768:12345-0@lo:400:kworker.0 Request processed in 91us (263us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318813.639975:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 740 00010000:00080000:2.0:1767318813.640133:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3100 x1853163982656768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318829 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318813.640156:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982656768:0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.645606:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982656896:0@lo:38:kworker.0 00000100:00100000:0.0:1767318813.645725:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.646887:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982656896 00000100:00100000:3.0:1767318813.646905:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982656896 00000100:00100000:3.0:1767318813.647047:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 967 00000100:00100000:3.0:1767318813.647056:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982656896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318813.647085:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767318813.651513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318813.651523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318813.651528:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318813.651539:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4057 00000100:00080000:1.0:1767318813.651545:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318813.651555:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318813.651572:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318813.651595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982657024:0@lo:400:kworker.0 00000100:00100000:1.0:1767318813.651691:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.653712:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982656896:12345-0@lo:38:kworker.0 Request processed in 6643us (7973us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318813.653730:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 967 00000100:00080000:0.0:1767318813.655893:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318813.655913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318813.655920:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982656896:0@lo:38:kworker.0 00000100:00100000:0.0:1767318813.656058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9880 x1853163982657152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318813.656513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318813.656521:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318813.656524:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318813.656531:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4057 00000100:00080000:0.0:1767318813.656537:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318813.656547:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318813.656570:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982657152:0@lo:38:kworker.0 00000100:00100000:0.0:1767318813.656690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab800 x1853163982657280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318813.656793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318813.656797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318813.656799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318813.656803:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4057 00000100:00080000:0.0:1767318813.656806:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318813.656810:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318813.656825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ffb80 x1853163982657664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318813.657684:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982657536:0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.660222:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318813.662209:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657536 00000100:00100000:0.0:1767318813.662236:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657536 00000100:00100000:0.0:1767318813.663781:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 483 00000100:00100000:0.0:1767318813.663806:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982657536:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.663970:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982657536:12345-0@lo:400:kworker.0 Request processed in 174us (3761us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318813.663978:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 483 00000100:00100000:0.0:1767318813.664064:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982657536:0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.664120:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318813.664135:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982657408:0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.664180:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318813.664210:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657408 00000100:00100000:0.0:1767318813.664217:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657408 00000100:00100000:0.0:1767318813.664247:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 484 00000100:00100000:0.0:1767318813.664253:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982657408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318813.664308:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982657408:12345-0@lo:400:kworker.0 Request processed in 55us (128us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318813.664314:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 484 00000100:00100000:0.0:1767318813.667666:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982657408:0@lo:400:kworker.0 00000100:00080000:0.0:1767318813.668456:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318813.668467:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318813.674459:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657024 00000100:00100000:3.0:1767318813.674493:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657024 00000100:00080000:3.0:1767318813.674693:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318813.674761:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 741 00000100:00100000:3.0:1767318813.674774:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982657024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318813.674927:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982657024:12345-0@lo:400:kworker.0 Request processed in 156us (23235us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318813.674936:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 741 00010000:00080000:1.0:1767318813.681127:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0380 x1853163982657024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318829 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318813.681153:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982657024:0@lo:400:kworker.0 00000100:00100000:2.0:1767318813.682250:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318813.682396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982657280:0@lo:38:kworker.0 00000100:00100000:3.0:1767318813.682417:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657152 00000100:00100000:3.0:1767318813.682431:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657152 00000100:00100000:2.0:1767318813.682489:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318813.682505:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968 00000100:00100000:3.0:1767318813.682516:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982657152:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318813.682521:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982657664:0@lo:38:kworker.0 00000100:00100000:2.0:1767318813.682555:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318813.682594:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657664 00000100:00100000:2.0:1767318813.682598:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657664 00000100:00100000:2.0:1767318813.682625:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969 00000100:00100000:2.0:1767318813.682632:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982657664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318813.682665:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318813.682721:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982657664:12345-0@lo:38:kworker.0 Request processed in 89us (164us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318813.682729:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969 00010000:00020000:3.0:1767318813.682754:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318813.682758:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657280 00000100:00100000:2.0:1767318813.682763:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657280 00000100:00100000:2.0:1767318813.682791:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 267 00000100:00100000:2.0:1767318813.682801:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982657280:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318813.682802:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982657152:12345-0@lo:38:kworker.0 Request processed in 283us (557us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318813.682815:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968 00010000:00020000:2.0:1767318813.682818:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318813.682850:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982657280:12345-0@lo:38:kworker.0 Request processed in 53us (360us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318813.682857:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 267 00000100:00080000:3.0:1767318813.682888:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.682896:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.682901:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982657280:0@lo:38:kworker.0 00000100:00080000:3.0:1767318813.682913:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.682916:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.682919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982657664:0@lo:38:kworker.0 00000100:00080000:3.0:1767318813.682927:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318813.682930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318813.682933:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982657152:0@lo:38:kworker.0 00010000:00010000:1.0:1767318813.829024:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318814.148589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982657792:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318814.148692:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318814.148871:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657792 00000100:00100000:3.0:1767318814.148884:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657792 00000100:00100000:3.0:1767318814.148955:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 514 00000100:00100000:3.0:1767318814.148966:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982657792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318814.149009:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318814.149382:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318814.149398:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982657920:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318814.149437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318814.149466:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982657920 00000100:00100000:1.0:1767318814.149471:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982657920 00000100:00100000:1.0:1767318814.149504:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 515 00000100:00100000:1.0:1767318814.149511:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982657920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318814.149533:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318814.149566:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982657920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (128us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318814.149574:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 515 00000100:00100000:1.0:1767318814.149638:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982657920:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318814.155141:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982657792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 6157us (6430us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318814.155163:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 514 00000100:00100000:2.0:1767318814.160439:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982657792:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318814.852492:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318815.630956:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318815.631013:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318815.631104:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590016 00000100:00100000:3.0:1767318815.631114:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590016 00000100:00100000:3.0:1767318815.631196:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915589888 00000100:00100000:3.0:1767318815.631200:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915589888 00000100:00100000:3.0:1767318815.631228:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 742 00000100:00100000:3.0:1767318815.631239:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915590016:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318815.631374:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915590016:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 137us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318815.631382:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 742 00000100:00100000:3.0:1767318815.631400:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 970 00000100:00100000:3.0:1767318815.631405:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915589888:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318815.631423:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318815.631448:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915589888:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (434us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318815.631455:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 970 00000100:00080000:0.0:1767318815.631751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318815.878711:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318816.900535:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318817.924710:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318818.756804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318818.756816:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318818.756892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318818.756897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318818.756901:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318818.756908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4062 00000100:00080000:2.0:1767318818.756914:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318818.756921:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318818.756947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982658176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318818.756992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318818.756996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318818.756998:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318818.757002:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4062 00000100:00100000:3.0:1767318818.757015:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982658176:0@lo:38:kworker.0 00000100:00100000:3.0:1767318818.757103:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318818.757127:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318818.757142:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982658048:0@lo:400:kworker.0 00000100:00100000:3.0:1767318818.757159:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658176 00000100:00100000:3.0:1767318818.757168:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658176 00000100:00100000:0.0:1767318818.757180:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318818.757209:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658048 00000100:00100000:3.0:1767318818.757214:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658048 00000100:00100000:0.0:1767318818.757218:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982658304:0@lo:400:kworker.0 00000100:00100000:0.0:1767318818.757245:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318818.757369:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318818.757377:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318818.757416:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56a00 x1853163982658432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318818.757431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318818.757435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318818.757436:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318818.757450:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4062 00000100:00080000:2.0:1767318818.757453:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318818.757457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318818.757470:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982658560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318818.757527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318818.757531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318818.757533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318818.757537:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4062 00000100:00080000:2.0:1767318818.757539:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318818.757544:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318818.757554:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d880 x1853163982658944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318818.757580:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982658688:0@lo:400:kworker.0 00000100:00100000:2.0:1767318818.757612:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318818.757651:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658688 00000100:00100000:2.0:1767318818.757656:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658688 00000100:00100000:2.0:1767318818.757690:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 485 00000100:00100000:2.0:1767318818.757700:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982658688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318818.757751:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982658688:12345-0@lo:400:kworker.0 Request processed in 52us (138us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318818.757759:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 485 00000100:00100000:2.0:1767318818.757802:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982658688:0@lo:400:kworker.0 00000100:00100000:2.0:1767318818.757822:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982658816:0@lo:400:kworker.0 00000100:00100000:2.0:1767318818.757842:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318818.757862:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658816 00000100:00100000:2.0:1767318818.757865:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658816 00000100:00100000:2.0:1767318818.757885:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 486 00000100:00100000:2.0:1767318818.757889:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982658816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318818.757916:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982658816:12345-0@lo:400:kworker.0 Request processed in 27us (74us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318818.757922:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 486 00000100:00100000:2.0:1767318818.757941:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982658816:0@lo:400:kworker.0 00000100:00080000:0.0:1767318818.758282:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318818.758287:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318818.758289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318818.763826:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00100000:0.0:1767318818.763857:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982658048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318818.763979:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982658048:12345-0@lo:400:kworker.0 Request processed in 138us (6798us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318818.763988:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00100000:3.0:1767318818.763988:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658304 00000100:00100000:3.0:1767318818.764002:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658304 00000100:00100000:0.0:1767318818.764013:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971 00000100:00100000:0.0:1767318818.764019:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982658176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318818.764043:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318818.764076:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982658176:12345-0@lo:38:kworker.0 Request processed in 55us (6973us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318818.764082:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971 00010000:00080000:0.0:1767318818.764136:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960380 x1853163982658048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318834 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318818.764151:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982658048:0@lo:400:kworker.0 00000100:00100000:3.0:1767318818.764173:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00100000:3.0:1767318818.764182:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982658304:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318818.764208:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318818.764229:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982658432:0@lo:38:kworker.0 00000100:00100000:0.0:1767318818.764303:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318818.764324:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982658560:0@lo:38:kworker.0 00000100:00100000:0.0:1767318818.764349:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318818.764370:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982658944:0@lo:38:kworker.0 00000100:00100000:0.0:1767318818.764390:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318818.764412:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318818.764418:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318818.764422:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982658176:0@lo:38:kworker.0 00000100:00100000:0.0:1767318818.764454:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658560 00000100:00100000:0.0:1767318818.764458:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658560 00000100:00100000:3.0:1767318818.764463:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982658304:12345-0@lo:400:kworker.0 Request processed in 281us (7215us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318818.764473:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00000100:00100000:3.0:1767318818.764489:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658432 00000100:00100000:3.0:1767318818.764493:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658432 00000100:00100000:0.0:1767318818.764496:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 268 00000100:00100000:0.0:1767318818.764503:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982658560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318818.764613:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318818.764648:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982658560:12345-0@lo:38:kworker.0 Request processed in 144us (297us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318818.764655:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 268 00000100:00100000:3.0:1767318818.764824:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982658944 00000100:00100000:3.0:1767318818.764827:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982658944 00000100:00100000:3.0:1767318818.764838:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972 00000100:00100000:3.0:1767318818.764844:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982658432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318818.764855:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767318818.764860:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318818.764867:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318818.764872:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982658560:0@lo:38:kworker.0 00000100:00100000:3.0:1767318818.764889:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982658432:12345-0@lo:38:kworker.0 Request processed in 44us (585us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767318818.764894:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318818.764899:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318818.764902:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982658432:0@lo:38:kworker.0 00000100:00100000:0.0:1767318818.764921:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 973 00000100:00100000:0.0:1767318818.764926:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982658944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318818.764935:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318818.764966:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982658944:12345-0@lo:38:kworker.0 Request processed in 39us (575us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318818.764973:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 973 00000100:00080000:0.0:1767318818.764994:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318818.764999:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318818.765001:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982658944:0@lo:38:kworker.0 00010000:00080000:0.0:1767318818.765150:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fad80 x1853163982658304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318834 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318818.765163:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982658304:0@lo:400:kworker.0 00000100:00100000:3.0:1767318818.765313:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972 00010000:00010000:1.0:1767318818.950662:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318819.270081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982659072:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318819.270177:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318819.270229:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318819.270252:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982659200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318819.270313:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318819.270411:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659072 00000100:00100000:2.0:1767318819.270420:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659072 00000100:00100000:1.0:1767318819.270432:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659200 00000100:00100000:1.0:1767318819.270440:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659200 00000100:00100000:2.0:1767318819.270486:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 516 00000100:00100000:2.0:1767318819.270497:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982659072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318819.270532:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318819.270592:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982659072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (416us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318819.270600:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 516 00000100:00100000:2.0:1767318819.270619:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 517 00000100:00100000:2.0:1767318819.270625:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982659200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318819.270643:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318819.270770:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982659200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 145us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318819.270777:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 517 00000100:00100000:0.0:1767318819.270918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982659072:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318819.286749:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982659200:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318819.972782:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318820.762949:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318820.762991:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318820.763064:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590272 00000100:00100000:1.0:1767318820.763087:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590272 00000100:00100000:1.0:1767318820.763194:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590144 00000100:00100000:1.0:1767318820.763198:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590144 00000100:00100000:1.0:1767318820.763233:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 745 00000100:00100000:1.0:1767318820.763246:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915590272:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318820.763260:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 974 00000100:00100000:2.0:1767318820.763268:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915590144:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318820.763316:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318820.763366:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915590272:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 122us (424us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318820.763374:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 745 00000100:00080000:1.0:1767318820.767720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318820.769032:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915590144:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 5751us (6028us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318820.769048:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 974 00010000:00010000:1.0:1767318820.996527:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318822.026031:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318823.050254:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318823.876552:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318823.876563:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318823.876631:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318823.876636:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318823.876640:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318823.876649:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318823.876655:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318823.876661:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318823.876688:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2680 x1853163982659456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318823.876732:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318823.876736:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318823.876738:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318823.876741:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318823.876744:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318823.876747:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318823.876758:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0a80 x1853163982659712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318823.876767:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318823.876769:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318823.876771:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318823.876774:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318823.876777:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318823.876780:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318823.876791:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3b80 x1853163982659840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318823.876812:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318823.876815:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318823.876817:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318823.876821:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1767318823.876823:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318823.876826:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318823.876836:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1f80 x1853163982660224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318823.876905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982660096:0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.876980:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318823.877035:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318823.877041:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982659328:0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877067:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318823.877102:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659328 00000100:00100000:0.0:1767318823.877147:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659328 00000100:00100000:0.0:1767318823.877208:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00100000:0.0:1767318823.877219:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982659328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877293:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982659328:12345-0@lo:400:kworker.0 Request processed in 54us (202us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318823.877300:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00010000:00080000:0.0:1767318823.877338:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e1c00 x1853163982659328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318839 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318823.877350:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982659328:0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877372:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982659584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877394:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318823.877412:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318823.877418:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982659968:0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877443:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318823.877465:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659584 00000100:00100000:0.0:1767318823.877469:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659584 00000100:00100000:0.0:1767318823.877492:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000100:00100000:0.0:1767318823.877498:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982659584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318823.877528:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982659584:12345-0@lo:400:kworker.0 Request processed in 31us (133us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318823.877534:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00010000:00080000:0.0:1767318823.877555:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0e00 x1853163982659584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318839 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318823.877564:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982659584:0@lo:400:kworker.0 00000100:00100000:2.0:1767318823.877624:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660096 00000100:00100000:2.0:1767318823.877629:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660096 00000100:00100000:2.0:1767318823.877664:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659968 00000100:00100000:2.0:1767318823.877667:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659968 00000100:00100000:2.0:1767318823.877696:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 487 00000100:00100000:2.0:1767318823.877703:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982660096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318823.877720:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 488 00000100:00100000:3.0:1767318823.877729:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463934a850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982659968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318823.877745:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982660096:12345-0@lo:400:kworker.0 Request processed in 43us (765us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318823.877752:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 487 00000100:00100000:3.0:1767318823.877761:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463934a850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982659968:12345-0@lo:400:kworker.0 Request processed in 34us (318us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318823.877771:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 488 00000100:00100000:2.0:1767318823.877777:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982660096:0@lo:400:kworker.0 00000100:00100000:2.0:1767318823.877793:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982659456:0@lo:38:kworker.0 00000100:00100000:2.0:1767318823.877817:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318823.877833:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982659712:0@lo:38:kworker.0 00000100:00100000:2.0:1767318823.877853:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318823.877869:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982659840:0@lo:38:kworker.0 00000100:00100000:2.0:1767318823.877889:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318823.877921:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982660224:0@lo:38:kworker.0 00000100:00100000:2.0:1767318823.877937:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318823.877960:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659456 00000100:00100000:2.0:1767318823.877963:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659456 00000100:00100000:2.0:1767318823.877974:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659712 00000100:00100000:2.0:1767318823.877976:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659712 00000100:00100000:2.0:1767318823.877982:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660224 00000100:00100000:2.0:1767318823.877984:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660224 00000100:00100000:2.0:1767318823.877993:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975 00000100:00100000:2.0:1767318823.877997:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982659456:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318823.878014:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 976 00010000:00020000:2.0:1767318823.878016:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318823.878018:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982659712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318823.878026:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318823.878048:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982659712:12345-0@lo:38:kworker.0 Request processed in 29us (193us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318823.878054:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 976 00000100:00100000:3.0:1767318823.878065:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977 00000100:00100000:3.0:1767318823.878069:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982660224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318823.878078:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318823.878094:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982659968:0@lo:400:kworker.0 00000100:00080000:1.0:1767318823.878143:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318823.878147:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318823.878149:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318823.878196:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982659840 00000100:00100000:1.0:1767318823.878201:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982659840 00000100:00100000:1.0:1767318823.878224:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 269 00000100:00100000:1.0:1767318823.878229:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982659840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318823.878238:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318823.882085:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982659456:12345-0@lo:38:kworker.0 Request processed in 4063us (4244us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318823.882101:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975 00000100:00100000:3.0:1767318823.882125:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982660224:12345-0@lo:38:kworker.0 Request processed in 4053us (4185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318823.882132:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977 00000100:00080000:0.0:1767318823.882865:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318823.882873:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318823.882877:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982659456:0@lo:38:kworker.0 00000100:00080000:0.0:1767318823.882891:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318823.882894:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318823.882897:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982659712:0@lo:38:kworker.0 00000100:00100000:1.0:1767318823.882988:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982659840:12345-0@lo:38:kworker.0 Request processed in 4756us (5095us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318823.882998:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 269 00000100:00080000:0.0:1767318823.892511:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318823.892525:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318823.892530:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982659840:0@lo:38:kworker.0 00000100:00080000:0.0:1767318823.892546:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318823.892549:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318823.892551:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982660224:0@lo:38:kworker.0 00010000:00010000:1.0:1767318824.069556:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318824.388898:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318824.388922:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982660352:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318824.388999:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318824.391446:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982660480:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318824.391553:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318824.401932:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660352 00000100:00100000:1.0:1767318824.401949:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660352 00000100:00100000:1.0:1767318824.402020:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660480 00000100:00100000:1.0:1767318824.402024:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660480 00000100:00100000:1.0:1767318824.402049:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 518 00000100:00100000:1.0:1767318824.402059:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982660352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318824.402100:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318824.402206:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982660352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 147us (13206us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.402213:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 518 00000100:00100000:1.0:1767318824.402232:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 519 00000100:00100000:1.0:1767318824.402238:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982660480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318824.402254:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318824.402306:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982660480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (10752us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318824.402313:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 519 00000100:00100000:2.0:1767318824.402952:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982660480:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318824.403119:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982660352:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318825.092762:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318825.874900:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318825.874950:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318825.875057:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590400 00000100:00100000:3.0:1767318825.875073:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590400 00000100:00100000:3.0:1767318825.875142:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590528 00000100:00100000:3.0:1767318825.875147:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590528 00000100:00100000:3.0:1767318825.875215:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 748 00000100:00100000:3.0:1767318825.875228:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915590528:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318825.875233:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 978 00000100:00100000:1.0:1767318825.875244:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915590400:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318825.875317:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767318825.875373:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318825.875450:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915590528:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 223us (498us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318825.875458:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 748 00000100:00100000:1.0:1767318825.878374:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915590400:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3119us (3474us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318825.878389:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 978 00010000:00010000:1.0:1767318826.116481:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318827.141056:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318828.164538:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318828.996443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318828.996454:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318828.996521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318828.996526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318828.996530:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318828.996538:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1767318828.996543:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318828.996551:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318828.996558:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982660608:0@lo:400:kworker.0 00000100:00100000:2.0:1767318828.996575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982660736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318828.996622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318828.996626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318828.996629:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767318828.996635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318828.996638:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1767318828.996643:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318828.996647:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318828.996660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469fb80 x1853163982660992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318828.996670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318828.996673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318828.996675:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318828.996679:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1767318828.996682:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767318828.996686:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318828.996702:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e300 x1853163982661120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318828.996724:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660608 00000100:00100000:3.0:1767318828.996733:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660608 00000100:00080000:2.0:1767318828.996735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318828.996740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318828.996742:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318828.996746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1767318828.996750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767318828.996753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318828.996770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163982661504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318828.996793:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 749 00000100:00080000:2.0:1767318828.996796:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318828.996808:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318828.996818:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982661376:0@lo:400:kworker.0 00000100:00100000:3.0:1767318828.996818:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982660608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318828.996860:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318828.996890:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982660608:12345-0@lo:400:kworker.0 Request processed in 87us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318828.996899:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 749 00010000:00080000:2.0:1767318828.996904:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469e680 x1853163982660608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318844 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318828.996918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982660608:0@lo:400:kworker.0 00000100:00100000:3.0:1767318828.996933:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982661248:0@lo:400:kworker.0 00000100:00100000:3.0:1767318828.996963:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318828.996996:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318828.997002:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982660864:0@lo:400:kworker.0 00000100:00100000:3.0:1767318828.997025:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318828.997048:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660864 00000100:00100000:3.0:1767318828.997053:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660864 00000100:00100000:2.0:1767318828.997084:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661376 00000100:00100000:2.0:1767318828.997090:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661376 00000100:00100000:3.0:1767318828.997101:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00100000:3.0:1767318828.997112:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982660864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318828.997126:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661248 00000100:00100000:2.0:1767318828.997131:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661248 00000100:00100000:3.0:1767318828.997155:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982660864:12345-0@lo:400:kworker.0 Request processed in 46us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318828.997162:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 489 00000100:00100000:3.0:1767318828.997167:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00100000:2.0:1767318828.997174:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982661376:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318828.997186:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318828.997191:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318828.997194:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318828.997209:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 490 00000100:00100000:2.0:1767318828.997218:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982661376:12345-0@lo:400:kworker.0 Request processed in 46us (356us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318828.997220:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982661248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318828.997232:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 489 00000100:00100000:3.0:1767318828.997265:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982661248:12345-0@lo:400:kworker.0 Request processed in 48us (300us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318828.997299:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982661376:0@lo:400:kworker.0 00000100:00100000:3.0:1767318828.997313:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 490 00010000:00080000:2.0:1767318828.997344:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ca80 x1853163982660864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318844 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318828.997355:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982660864:0@lo:400:kworker.0 00000100:00100000:2.0:1767318828.997367:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982661248:0@lo:400:kworker.0 00000100:00100000:1.0:1767318828.997522:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982660736:0@lo:38:kworker.0 00000100:00100000:1.0:1767318828.997553:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318828.997587:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982660992:0@lo:38:kworker.0 00000100:00100000:0.0:1767318828.997610:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660736 00000100:00100000:1.0:1767318828.997614:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318828.997617:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660736 00000100:00100000:1.0:1767318828.997640:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982661120:0@lo:38:kworker.0 00000100:00100000:0.0:1767318828.997655:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982660992 00000100:00100000:0.0:1767318828.997661:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982660992 00000100:00100000:1.0:1767318828.997667:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318828.997687:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 979 00000100:00100000:0.0:1767318828.997693:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982660736:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318828.997700:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982661504:0@lo:38:kworker.0 00010000:00020000:0.0:1767318828.997712:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318828.997725:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318828.997760:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318828.997765:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318828.997768:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982660736:0@lo:38:kworker.0 00000100:00100000:3.0:1767318828.997795:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661504 00000100:00100000:1.0:1767318828.997797:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661120 00000100:00100000:1.0:1767318828.997802:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661120 00000100:00100000:3.0:1767318828.997811:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661504 00000100:00100000:3.0:1767318828.997828:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980 00000100:00100000:3.0:1767318828.997833:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982660992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318828.997844:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318828.997882:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 270 00000100:00100000:1.0:1767318828.997888:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982661120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318828.997900:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318828.997907:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981 00000100:00100000:2.0:1767318828.997913:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982661504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318828.997930:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318829.002219:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982660736:12345-0@lo:38:kworker.0 Request processed in 4513us (4652us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318829.002237:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 979 00000100:00100000:3.0:1767318829.008477:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982660992:12345-0@lo:38:kworker.0 Request processed in 10631us (10853us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318829.008493:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980 00000100:00080000:0.0:1767318829.008633:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318829.008639:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318829.008643:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982660992:0@lo:38:kworker.0 00000100:00100000:1.0:1767318829.015824:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982661120:12345-0@lo:38:kworker.0 Request processed in 17925us (18145us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318829.015840:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 270 00000100:00080000:3.0:1767318829.016077:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318829.016083:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318829.016087:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982661120:0@lo:38:kworker.0 00000100:00100000:2.0:1767318829.016868:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982661504:12345-0@lo:38:kworker.0 Request processed in 18948us (19138us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318829.016880:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981 00000100:00080000:3.0:1767318829.017878:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318829.017889:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318829.017894:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982661504:0@lo:38:kworker.0 00010000:00010000:1.0:1767318829.193168:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318829.517213:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982661632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318829.517319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318829.517425:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318829.517435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982661760:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318829.517462:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318829.517493:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661632 00000100:00100000:2.0:1767318829.517500:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661632 00000100:00100000:2.0:1767318829.517540:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661760 00000100:00100000:2.0:1767318829.517542:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661760 00000100:00100000:2.0:1767318829.517559:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 520 00000100:00100000:2.0:1767318829.517567:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982661632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318829.517599:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318829.517648:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982661632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (330us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318829.517668:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 520 00000100:00100000:2.0:1767318829.518015:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 521 00000100:00100000:2.0:1767318829.518023:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982661760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318829.518040:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318829.518083:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982661760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (620us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318829.518089:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 521 00000100:00100000:2.0:1767318829.518128:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982661760:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318829.518160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982661632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767318830.214675:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318830.990454:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318830.990760:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590656 00000100:00100000:3.0:1767318830.990771:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590656 00000100:00100000:3.0:1767318830.990821:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 982 00000100:00100000:3.0:1767318830.990828:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915590656:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318830.990851:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318830.990952:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915590656:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 123us (511us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318830.990958:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 982 00000100:00100000:0.0:1767318830.999643:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318830.999918:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915590784 00000100:00100000:3.0:1767318830.999928:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915590784 00000100:00080000:2.0:1767318831.000064:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318831.000114:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 751 00000100:00100000:3.0:1767318831.000125:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915590784:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318831.000376:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915590784:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 252us (744us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318831.000385:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 751 00010000:00010000:1.0:1767318831.244613:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318832.261390:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318833.284534:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318834.116493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318834.116504:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318834.116589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982661888:0@lo:400:kworker.0 00000100:00100000:2.0:1767318834.116663:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318834.116745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318834.116749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318834.116753:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318834.116759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4077 00000100:00080000:0.0:1767318834.116763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318834.116770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318834.116793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982662016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318834.116837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318834.116840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318834.116843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318834.116847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4077 00000100:00080000:0.0:1767318834.116851:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1767318834.116855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318834.116869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163982662272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318834.116869:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982662016:0@lo:38:kworker.0 00000100:00080000:0.0:1767318834.116885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318834.116889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318834.116891:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318834.116895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4077 00000100:00080000:0.0:1767318834.116898:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767318834.116901:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318834.116912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fca80 x1853163982662400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318834.116929:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318834.117006:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982662272:0@lo:38:kworker.0 00000100:00100000:1.0:1767318834.117031:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318834.117049:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982662400:0@lo:38:kworker.0 00000100:00100000:1.0:1767318834.117075:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318834.117116:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982661888 00000100:00100000:1.0:1767318834.117124:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982661888 00000100:00100000:1.0:1767318834.117175:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662016 00000100:00100000:1.0:1767318834.117178:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662016 00000100:00100000:1.0:1767318834.117192:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662272 00000100:00100000:1.0:1767318834.117195:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662272 00000100:00100000:1.0:1767318834.117207:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00100000:1.0:1767318834.117218:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982661888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117478:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983 00000100:00100000:3.0:1767318834.117484:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982662016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318834.117502:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318834.117541:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982662016:12345-0@lo:38:kworker.0 Request processed in 56us (616us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318834.117548:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983 00000100:00100000:3.0:1767318834.117567:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984 00000100:00100000:3.0:1767318834.117571:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982662272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318834.117580:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318834.117603:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982662272:12345-0@lo:38:kworker.0 Request processed in 30us (571us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318834.117609:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984 00000100:00100000:3.0:1767318834.117640:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982662528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117668:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.117689:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318834.117695:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982662144:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117716:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.117739:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662528 00000100:00100000:3.0:1767318834.117743:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662528 00000100:00100000:3.0:1767318834.117771:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 491 00000100:00100000:3.0:1767318834.117777:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982662528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117811:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982662528:12345-0@lo:400:kworker.0 Request processed in 34us (141us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318834.117817:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 491 00000100:00100000:3.0:1767318834.117856:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982662528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117876:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662144 00000100:00100000:3.0:1767318834.117879:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662144 00000100:00100000:3.0:1767318834.117896:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 753 00000100:00100000:3.0:1767318834.117901:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982662144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.117928:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982662144:12345-0@lo:400:kworker.0 Request processed in 27us (211us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318834.117933:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 753 00010000:00080000:3.0:1767318834.117952:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc000 x1853163982662144/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318850 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318834.119475:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982662144:0@lo:400:kworker.0 00000100:00080000:3.0:1767318834.119541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318834.119547:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318834.119565:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662400 00000100:00100000:3.0:1767318834.119572:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662400 00000100:00100000:3.0:1767318834.119609:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 271 00000100:00100000:3.0:1767318834.119616:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982662400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318834.119633:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318834.119691:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982662400:12345-0@lo:38:kworker.0 Request processed in 74us (2615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318834.119698:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 271 00000100:00100000:3.0:1767318834.119719:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982662656:0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.119750:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.119781:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662656 00000100:00100000:3.0:1767318834.119787:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662656 00000100:00100000:3.0:1767318834.119821:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 492 00000100:00100000:3.0:1767318834.119827:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982662656:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318834.119865:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982662656:12345-0@lo:400:kworker.0 Request processed in 39us (114us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318834.119871:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 492 00000100:00080000:3.0:1767318834.119884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318834.119917:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982662656:0@lo:400:kworker.0 00000100:00080000:3.0:1767318834.121514:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318834.121525:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318834.121529:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982662272:0@lo:38:kworker.0 00000100:00080000:3.0:1767318834.121542:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318834.121546:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318834.121548:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982662400:0@lo:38:kworker.0 00000100:00080000:3.0:1767318834.121560:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318834.121564:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318834.121567:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982662016:0@lo:38:kworker.0 00000100:00080000:0.0:1767318834.121598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318834.121605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318834.121609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318834.121614:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4077 00000100:00080000:0.0:1767318834.121619:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1767318834.121626:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318834.121661:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ffb80 x1853163982662784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318834.121713:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982661888:12345-0@lo:400:kworker.0 Request processed in 4494us (5056us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318834.121730:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00100000:3.0:1767318834.121829:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982662784:0@lo:38:kworker.0 00000100:00100000:3.0:1767318834.121889:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767318834.121959:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe680 x1853163982661888/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318850 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318834.121984:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982661888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318834.132073:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662784 00000100:00100000:1.0:1767318834.132095:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662784 00000100:00100000:1.0:1767318834.132170:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 985 00000100:00100000:1.0:1767318834.132181:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982662784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318834.132210:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318834.132347:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982662784:12345-0@lo:38:kworker.0 Request processed in 167us (10457us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318834.132355:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 985 00000100:00080000:2.0:1767318834.135931:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318834.135944:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318834.135949:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982662784:0@lo:38:kworker.0 00010000:00010000:1.0:1767318834.309223:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318834.628648:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982662912:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318834.628746:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.628999:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982663040:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318834.629027:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318834.629103:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982662912 00000100:00100000:3.0:1767318834.629111:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982662912 00000100:00100000:3.0:1767318834.629156:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663040 00000100:00100000:3.0:1767318834.629159:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663040 00000100:00100000:3.0:1767318834.629175:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 522 00000100:00100000:3.0:1767318834.629183:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982662912:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318834.629294:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 523 00000100:00100000:1.0:1767318834.629304:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982663040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318834.629332:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318834.629378:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982663040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318834.629385:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 523 00000100:00100000:1.0:1767318834.629438:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982663040:0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318834.629498:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318834.629526:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982662912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 344us (782us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318834.629533:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 522 00000100:00100000:2.0:1767318834.629687:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982662912:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318835.089247:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767318835.089323:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915590912 offset 0 length 520: 4 00000100:00100000:2.0:1767318835.089347:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318835.089436:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591168 00000100:00100000:1.0:1767318835.089445:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591168 00000100:00100000:1.0:1767318835.089521:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591040 00000100:00100000:1.0:1767318835.089524:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591040 00000100:00100000:1.0:1767318835.089541:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 754 00000100:00100000:1.0:1767318835.089551:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318835.089647:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318835.089660:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 754 00000100:00100000:1.0:1767318835.089772:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 986 00000100:00100000:1.0:1767318835.089777:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915591040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318835.089801:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318835.089830:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915591040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (482us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318835.089837:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 986 00000100:00080000:0.0:1767318835.089905:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318835.335976:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318836.357572:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318837.380797:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318838.212476:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318813/real 1767318813] req@ffff93463a8e2a00 x1853163982656640/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318838 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318838.212519:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e2a00 x1853163982656640/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318838 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767318838.212544:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318838.212569:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318838.212574:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982656640:0@lo:250:kworker.0 00000100:00080000:1.0:1767318838.212616:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318838.212624:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318838.212628:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318838.212635:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4062 00000100:00080000:1.0:1767318838.212640:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318838.212646:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318838.212684:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0e00 x1853163982663168/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318838.212764:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982663168:0@lo:250:kworker.0 00000400:00000100:2.0:1767318838.212833:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982663168 offset 0 length 520: 4 00000100:00080000:1.0:1767318838.212946:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318838.212958:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318838.212960:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318838.212964:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4082 00000100:00080000:1.0:1767318838.212969:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1767318838.212974:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318838.212991:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318838.212991:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982663424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318838.213007:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982663296:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.213059:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318838.213253:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663296 00000100:00100000:3.0:1767318838.213262:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663296 00000100:00100000:2.0:1767318838.213377:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982663424:0@lo:38:kworker.0 00000100:00100000:2.0:1767318838.213405:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767318838.213692:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318838.213696:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318838.213697:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318838.213701:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4082 00000100:00080000:1.0:1767318838.213705:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1767318838.213709:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318838.213725:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163982663680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318838.213735:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318838.213749:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318838.213750:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318838.213754:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4082 00000100:00080000:1.0:1767318838.213756:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318838.213760:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318838.213772:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35500 x1853163982663808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318838.213799:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318838.213803:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318838.213805:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318838.213809:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4082 00000100:00080000:1.0:1767318838.213811:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767318838.213814:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318838.213826:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163982664192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318838.213846:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982663552:0@lo:400:kworker.0 00000100:00080000:0.0:1767318838.213873:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318838.213881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318838.214067:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982663936:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.214108:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318838.214146:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982664064:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.214178:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318838.217944:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663424 00000100:00100000:2.0:1767318838.218035:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663424 00000100:00100000:2.0:1767318838.218353:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663552 00000100:00100000:2.0:1767318838.218358:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663552 00000100:00100000:0.0:1767318838.218405:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 755 00000100:00100000:2.0:1767318838.218481:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 756 00000100:00100000:0.0:1767318838.218487:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982663296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318838.218494:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982663552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318838.218580:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982663552:12345-0@lo:400:kworker.0 Request processed in 91us (4700us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318838.218588:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 756 00000100:00100000:0.0:1767318838.218595:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982663296:12345-0@lo:400:kworker.0 Request processed in 120us (5535us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.218606:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 755 00000100:00100000:0.0:1767318838.218632:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987 00000100:00100000:0.0:1767318838.218636:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982663424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318838.218657:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318838.218687:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982663424:12345-0@lo:38:kworker.0 Request processed in 49us (5281us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318838.218693:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987 00010000:00080000:0.0:1767318838.218776:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e1c00 x1853163982663296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318854 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318838.218790:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982663296:0@lo:400:kworker.0 00010000:00080000:0.0:1767318838.218837:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e2680 x1853163982663552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318854 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318838.218847:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982663552:0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.218879:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663936 00000100:00100000:0.0:1767318838.218884:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663936 00000100:00100000:0.0:1767318838.218927:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664064 00000100:00100000:0.0:1767318838.218931:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664064 00000100:00100000:2.0:1767318838.218945:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 493 00000100:00100000:2.0:1767318838.219037:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982663936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318838.219415:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982663936:12345-0@lo:400:kworker.0 Request processed in 461us (5306us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.225378:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 494 00000100:00100000:2.0:1767318838.225476:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 493 00000100:00080000:3.0:1767318838.225512:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318838.225519:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318838.225523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318838.225549:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982663680:0@lo:38:kworker.0 00000100:00100000:2.0:1767318838.225566:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982663936:0@lo:400:kworker.0 00000100:00100000:3.0:1767318838.225638:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318838.225683:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982663808:0@lo:38:kworker.0 00000100:00100000:3.0:1767318838.225711:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318838.225730:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982664192:0@lo:38:kworker.0 00000100:00100000:3.0:1767318838.225749:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318838.225764:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663808 00000100:00100000:1.0:1767318838.225774:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663808 00000100:00080000:3.0:1767318838.225777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318838.225783:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318838.225787:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982663424:0@lo:38:kworker.0 00000100:00100000:1.0:1767318838.225812:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 272 00000100:00100000:1.0:1767318838.225820:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982663808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318838.225840:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318838.225905:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982663808:12345-0@lo:38:kworker.0 Request processed in 84us (191us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318838.225912:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 272 00000100:00100000:1.0:1767318838.225935:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982663680 00000100:00100000:1.0:1767318838.225939:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982663680 00000100:00100000:1.0:1767318838.225957:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664192 00000100:00100000:1.0:1767318838.225960:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664192 00000100:00100000:0.0:1767318838.226021:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982664064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318838.226074:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982664064:12345-0@lo:400:kworker.0 Request processed in 94us (11895us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318838.226092:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 494 00000100:00100000:0.0:1767318838.226128:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982664064:0@lo:400:kworker.0 00000100:00080000:3.0:1767318838.226171:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318838.226175:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318838.226178:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982663808:0@lo:38:kworker.0 00000100:00100000:1.0:1767318838.226529:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988 00000100:00100000:1.0:1767318838.226536:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982663680:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318838.226544:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989 00010000:00020000:1.0:1767318838.226550:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318838.226552:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982664192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318838.226576:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318838.226587:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982663680:12345-0@lo:38:kworker.0 Request processed in 50us (949us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318838.226595:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988 00000100:00100000:3.0:1767318838.226614:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982664192:12345-0@lo:38:kworker.0 Request processed in 62us (862us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318838.226622:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989 00000100:00080000:3.0:1767318838.226649:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318838.226654:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318838.226657:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982663680:0@lo:38:kworker.0 00000100:00080000:3.0:1767318838.226665:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318838.226668:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318838.226671:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982664192:0@lo:38:kworker.0 00010000:00010000:1.0:1767318838.404494:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318839.435112:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318839.748550:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318839.748577:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982664320:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318839.748645:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318839.748738:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664320 00000100:00100000:1.0:1767318839.748746:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664320 00000100:00100000:1.0:1767318839.748800:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 524 00000100:00100000:1.0:1767318839.748830:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982664320:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318839.748873:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318839.748922:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982664320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (277us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318839.748930:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 524 00000100:00100000:1.0:1767318839.748974:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982664320:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318839.749443:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318839.749454:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982664448:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318839.749489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318839.749831:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664448 00000100:00100000:3.0:1767318839.749837:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664448 00000100:00100000:3.0:1767318839.749912:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 525 00000100:00100000:3.0:1767318839.749919:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982664448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318839.749940:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318839.749983:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982664448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (493us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318839.749990:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 525 00000100:00100000:2.0:1767318839.750062:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982664448:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318840.203451:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318840.203706:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591296 00000100:00100000:0.0:1767318840.203717:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591296 00000100:00100000:0.0:1767318840.203769:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 990 00000100:00100000:0.0:1767318840.203777:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915591296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318840.203801:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318840.203899:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915591296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 120us (460us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318840.203907:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 990 00000100:00100000:2.0:1767318840.207560:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318840.208474:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591424 00000100:00100000:3.0:1767318840.208488:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591424 00000100:00100000:3.0:1767318840.208578:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 757 00000100:00100000:3.0:1767318840.208588:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915591424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318840.208678:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915591424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (1142us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318840.208686:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 757 00000100:00080000:0.0:1767318840.209205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318840.452563:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318841.476638:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318842.501592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318843.332518:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318843.332530:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318843.332607:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318843.332611:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318843.332615:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318843.332623:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4086 00000100:00080000:3.0:1767318843.332629:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767318843.332635:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318843.332643:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982664576:0@lo:400:kworker.0 00000100:00100000:3.0:1767318843.332660:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90000 x1853163982664704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318843.332709:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767318843.332713:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318843.332715:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318843.332719:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318843.332723:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4086 00000100:00080000:3.0:1767318843.332726:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767318843.332730:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318843.332742:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982664960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318843.332751:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318843.332755:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318843.332756:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767318843.332822:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982664704:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.332855:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.332885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982664960:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.332902:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.332945:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664576 00000100:00100000:1.0:1767318843.332953:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664576 00000100:00100000:1.0:1767318843.333001:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664704 00000100:00100000:1.0:1767318843.333004:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664704 00000100:00100000:1.0:1767318843.333017:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664960 00000100:00100000:1.0:1767318843.333020:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664960 00000100:00100000:1.0:1767318843.333031:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 758 00000100:00100000:1.0:1767318843.333041:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982664576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318843.333231:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318843.333242:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982664832:0@lo:400:kworker.0 00000100:00100000:0.0:1767318843.333299:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318843.333321:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318843.333332:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982664832 00000100:00100000:0.0:1767318843.333335:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982664832 00000100:00100000:0.0:1767318843.333365:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 759 00000100:00100000:0.0:1767318843.333371:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982664832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318843.333407:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982664832:12345-0@lo:400:kworker.0 Request processed in 36us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318843.333414:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 759 00000100:00100000:0.0:1767318843.333429:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767318843.333434:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982664704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318843.333450:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318843.333482:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982664704:12345-0@lo:38:kworker.0 Request processed in 46us (625us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318843.333488:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767318843.333502:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992 00000100:00100000:0.0:1767318843.333507:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982664960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318843.333516:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318843.333539:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982664960:12345-0@lo:38:kworker.0 Request processed in 31us (636us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318843.333544:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992 00000100:00080000:0.0:1767318843.333559:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318843.333583:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f800 x1853163982664832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318859 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318843.333594:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982664832:0@lo:400:kworker.0 00000100:00080000:0.0:1767318843.333633:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.333638:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.333641:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982664704:0@lo:38:kworker.0 00000100:00080000:0.0:1767318843.333648:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.333651:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.333654:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982664960:0@lo:38:kworker.0 00000100:00100000:1.0:1767318843.333809:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982664576:12345-0@lo:400:kworker.0 Request processed in 769us (1098us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318843.333816:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 758 00010000:00080000:2.0:1767318843.338830:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d91f80 x1853163982664576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318859 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318843.338848:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982664576:0@lo:400:kworker.0 00000100:00080000:3.0:1767318843.339627:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4086 00000100:00080000:3.0:1767318843.339634:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318843.339641:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318843.339682:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982665088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318843.339762:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982665088:0@lo:38:kworker.0 00000100:00100000:0.0:1767318843.339846:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318843.340031:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318843.340035:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318843.340038:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318843.340042:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4086 00000100:00080000:3.0:1767318843.340045:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318843.340049:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318843.340061:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625500 x1853163982665472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318843.340119:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982665216:0@lo:400:kworker.0 00000100:00100000:2.0:1767318843.340156:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318843.340196:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318843.340203:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982665344:0@lo:400:kworker.0 00000100:00100000:2.0:1767318843.340228:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318843.340440:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665088 00000100:00100000:1.0:1767318843.340448:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665088 00000100:00100000:1.0:1767318843.340497:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 273 00000100:00100000:1.0:1767318843.340505:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982665088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318843.340525:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318843.340625:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665216 00000100:00100000:2.0:1767318843.340630:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665216 00000100:00100000:2.0:1767318843.340668:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665344 00000100:00100000:2.0:1767318843.340671:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665344 00000100:00100000:2.0:1767318843.340689:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 495 00000100:00100000:2.0:1767318843.340695:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982665216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318843.340739:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982665216:12345-0@lo:400:kworker.0 Request processed in 43us (582us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318843.340746:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 495 00000100:00100000:2.0:1767318843.340760:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 496 00000100:00100000:2.0:1767318843.340765:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982665344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318843.340809:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982665344:12345-0@lo:400:kworker.0 Request processed in 43us (580us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318843.340814:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 496 00000100:00100000:2.0:1767318843.340856:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982665216:0@lo:400:kworker.0 00000100:00100000:2.0:1767318843.340894:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982665344:0@lo:400:kworker.0 00000100:00080000:2.0:1767318843.340930:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318843.340934:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318843.344697:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982665472:0@lo:38:kworker.0 00000100:00100000:0.0:1767318843.344774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318843.344902:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665472 00000100:00100000:2.0:1767318843.344918:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665472 00000100:00100000:2.0:1767318843.344960:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993 00000100:00100000:2.0:1767318843.344967:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982665472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318843.344988:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318843.345029:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982665472:12345-0@lo:38:kworker.0 Request processed in 61us (255us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318843.345036:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993 00000100:00100000:1.0:1767318843.355575:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982665088:12345-0@lo:38:kworker.0 Request processed in 15058us (15717us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318843.355602:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 273 00000100:00080000:0.0:1767318843.359528:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.359541:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.359547:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982665088:0@lo:38:kworker.0 00000100:00080000:0.0:1767318843.359598:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318843.359602:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318843.359606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982665472:0@lo:38:kworker.0 00010000:00010000:1.0:1767318843.526448:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318844.548631:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318844.868748:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982665600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318844.868852:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318844.876452:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665600 00000100:00100000:1.0:1767318844.876468:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665600 00000100:00100000:1.0:1767318844.876546:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 526 00000100:00100000:1.0:1767318844.876557:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982665600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318844.876592:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318844.876671:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982665600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 116us (7819us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318844.876678:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 526 00000100:00100000:0.0:1767318844.877494:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982665600:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318844.877980:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982665728:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318844.878025:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318844.878479:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665728 00000100:00100000:3.0:1767318844.878486:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665728 00000100:00100000:3.0:1767318844.878535:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 527 00000100:00100000:3.0:1767318844.878545:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982665728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318844.878569:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318844.878617:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982665728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (591us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318844.878624:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 527 00000100:00100000:0.0:1767318844.878798:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982665728:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318845.324670:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318845.324717:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318845.324821:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591680 00000100:00100000:3.0:1767318845.324831:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591680 00000100:00100000:3.0:1767318845.324906:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591552 00000100:00100000:3.0:1767318845.324910:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591552 00000100:00100000:3.0:1767318845.324933:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 760 00000100:00100000:3.0:1767318845.324943:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591680:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318845.325016:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591680:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (358us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318845.325024:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 760 00000100:00100000:3.0:1767318845.325050:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 994 00000100:00100000:3.0:1767318845.325054:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915591552:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318845.325073:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318845.325098:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915591552:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (382us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318845.325104:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 994 00000100:00080000:0.0:1767318845.325653:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318845.572535:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318846.596514:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318847.620581:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318848.452399:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318848.452410:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318848.452482:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318848.452486:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318848.452490:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318848.452509:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4091 00000100:00080000:3.0:1767318848.452515:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767318848.452523:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318848.452547:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163982665984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318848.452592:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318848.452596:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318848.452597:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318848.452601:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4091 00000100:00080000:3.0:1767318848.452604:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318848.452607:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318848.452620:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627100 x1853163982666240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318848.452634:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318848.452637:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318848.452638:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318848.452642:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4091 00000100:00080000:3.0:1767318848.452644:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318848.452647:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318848.452658:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625500 x1853163982666368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318848.452682:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318848.452685:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318848.452687:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318848.452690:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4091 00000100:00080000:3.0:1767318848.452693:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318848.452707:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318848.452721:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163982666752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318848.452731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982665856:0@lo:400:kworker.0 00000100:00100000:1.0:1767318848.452804:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318848.452819:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982665984:0@lo:38:kworker.0 00000100:00100000:0.0:1767318848.452895:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982666624:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.452931:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318848.452985:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982666112:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.453005:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982666496:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.453020:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318848.453123:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666624 00000100:00100000:2.0:1767318848.453129:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666624 00000100:00100000:2.0:1767318848.453183:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666496 00000100:00100000:2.0:1767318848.453186:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666496 00000100:00100000:2.0:1767318848.453208:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 497 00000100:00100000:2.0:1767318848.453217:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982666624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318848.453260:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982666624:12345-0@lo:400:kworker.0 Request processed in 44us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318848.453266:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 497 00000100:00100000:2.0:1767318848.453298:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 498 00000100:00100000:2.0:1767318848.453302:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982666496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318848.453319:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982666496:12345-0@lo:400:kworker.0 Request processed in 18us (299us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318848.453323:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 498 00000100:00100000:2.0:1767318848.453496:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318848.453542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318848.453545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318848.453569:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982666496:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.453587:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982666624:0@lo:400:kworker.0 00000100:00100000:1.0:1767318848.453938:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318848.454815:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665856 00000100:00100000:3.0:1767318848.454825:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665856 00000100:00100000:3.0:1767318848.454874:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666112 00000100:00100000:3.0:1767318848.454877:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666112 00000100:00100000:3.0:1767318848.454912:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 761 00000100:00100000:3.0:1767318848.454918:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982665856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318848.454977:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982665856:12345-0@lo:400:kworker.0 Request processed in 58us (2173us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318848.454984:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 761 00000100:00100000:3.0:1767318848.454998:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 762 00000100:00100000:3.0:1767318848.455004:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982666112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318848.455034:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982666112:12345-0@lo:400:kworker.0 Request processed in 30us (1537us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318848.455039:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 762 00010000:00080000:3.0:1767318848.455081:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625c00 x1853163982665856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318864 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318848.455094:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982665856:0@lo:400:kworker.0 00000100:00080000:2.0:1767318848.455179:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318848.455202:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626a00 x1853163982666112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318864 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318848.455211:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982666112:0@lo:400:kworker.0 00000100:00100000:0.0:1767318848.457353:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982665984 00000100:00100000:0.0:1767318848.457364:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982665984 00000100:00100000:0.0:1767318848.457510:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995 00000100:00100000:0.0:1767318848.457518:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982665984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318848.457541:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318848.457599:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982665984:12345-0@lo:38:kworker.0 Request processed in 81us (3661us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318848.457606:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995 00000100:00080000:1.0:1767318848.457675:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.457681:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.457685:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982665984:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.457704:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982666240:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.457740:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.457761:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982666368:0@lo:38:kworker.0 00000100:00100000:0.0:1767318848.457773:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666240 00000100:00100000:0.0:1767318848.457777:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666240 00000100:00100000:1.0:1767318848.457868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318848.457906:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982666752:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.457928:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318848.458016:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666368 00000100:00100000:2.0:1767318848.458021:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666368 00000100:00100000:2.0:1767318848.458044:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 274 00000100:00100000:2.0:1767318848.458049:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982666368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318848.458060:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318848.458088:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982666368:12345-0@lo:38:kworker.0 Request processed in 38us (220us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318848.458094:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 274 00000100:00100000:0.0:1767318848.461366:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666752 00000100:00100000:0.0:1767318848.461385:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666752 00000100:00080000:1.0:1767318848.461402:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.461412:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.461417:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982666368:0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.461471:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996 00000100:00100000:1.0:1767318848.461480:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982666240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318848.461502:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318848.461573:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982666240:12345-0@lo:38:kworker.0 Request processed in 93us (3831us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318848.461574:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 997 00000100:00100000:0.0:1767318848.461579:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982666752:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318848.461580:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996 00010000:00020000:0.0:1767318848.461588:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767318848.461610:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318848.461615:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.461618:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982666240:0@lo:38:kworker.0 00000100:00080000:1.0:1767318848.461627:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767318848.461630:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982666752:12345-0@lo:38:kworker.0 Request processed in 50us (3701us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767318848.461631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318848.461634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982666752:0@lo:38:kworker.0 00000100:00100000:0.0:1767318848.461636:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 997 00010000:00010000:1.0:1767318848.647877:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318849.669420:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318849.988593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982666880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318849.988679:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318849.988931:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982666880 00000100:00100000:0.0:1767318849.988940:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982666880 00000100:00100000:0.0:1767318849.988999:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 528 00000100:00100000:0.0:1767318849.989010:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982666880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318849.989043:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318849.989098:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982666880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (418us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318849.989105:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 528 00000100:00100000:1.0:1767318849.989227:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982666880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318849.991101:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982667008:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318849.991162:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318849.991341:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667008 00000100:00100000:1.0:1767318849.991349:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667008 00000100:00100000:1.0:1767318849.991393:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 529 00000100:00100000:1.0:1767318849.991402:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982667008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318849.991428:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318849.991471:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982667008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318849.991478:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 529 00000100:00100000:2.0:1767318849.991856:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982667008:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318850.445150:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318850.445235:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318850.445318:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591936 00000100:00100000:2.0:1767318850.445327:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591936 00000100:00100000:2.0:1767318850.445392:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915591808 00000100:00100000:2.0:1767318850.445395:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915591808 00000100:00100000:2.0:1767318850.445411:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 763 00000100:00100000:2.0:1767318850.445419:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591936:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318850.445487:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915591936:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 69us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318850.445495:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 763 00000100:00100000:2.0:1767318850.445512:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 998 00000100:00100000:2.0:1767318850.445516:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915591808:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318850.445535:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318850.445558:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915591808:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (323us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318850.445564:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 998 00000100:00080000:0.0:1767318850.445706:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318850.693052:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318851.716505:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318852.740663:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318853.574804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318853.574814:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318853.574879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318853.574882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318853.574886:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318853.574894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4096 00000100:00080000:2.0:1767318853.574899:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318853.574905:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318853.574928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320a80 x1853163982667264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318853.574981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318853.574985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318853.574987:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318853.574991:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4096 00000100:00080000:2.0:1767318853.574993:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318853.574997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318853.575010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982667520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318853.575136:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982667136:0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.575204:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318853.575304:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667136 00000100:00100000:0.0:1767318853.575311:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667136 00000100:00100000:0.0:1767318853.575387:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 764 00000100:00100000:0.0:1767318853.575397:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982667136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.575449:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982667136:12345-0@lo:400:kworker.0 Request processed in 53us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318853.575459:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 764 00000100:00080000:0.0:1767318853.575481:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318853.575523:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f800 x1853163982667136/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318869 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318853.575536:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982667136:0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.575575:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982667392:0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.575610:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318853.575639:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667392 00000100:00100000:0.0:1767318853.575643:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667392 00000100:00100000:0.0:1767318853.575669:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 765 00000100:00100000:0.0:1767318853.575913:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982667392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318853.575955:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982667392:12345-0@lo:400:kworker.0 Request processed in 280us (354us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318853.575962:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 765 00000100:00080000:0.0:1767318853.575976:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318853.575994:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320380 x1853163982667392/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318869 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318853.576003:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982667392:0@lo:400:kworker.0 00000100:00100000:1.0:1767318853.576184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982667264:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.576218:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318853.576298:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667264 00000100:00100000:0.0:1767318853.576302:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667264 00000100:00100000:0.0:1767318853.576330:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999 00000100:00100000:0.0:1767318853.576335:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982667264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318853.577455:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318853.581736:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982667264:12345-0@lo:38:kworker.0 Request processed in 5387us (5503us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318853.581751:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999 00000100:00080000:2.0:1767318853.596518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318853.596528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318853.596531:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318853.596539:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4096 00000100:00080000:2.0:1767318853.596544:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318853.596551:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318853.596622:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163982667648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318853.596695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318853.596700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318853.596701:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318853.596705:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4096 00000100:00080000:2.0:1767318853.596708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318853.596711:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318853.596722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359880 x1853163982668032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318853.596769:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318853.596783:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982667904:0@lo:400:kworker.0 00000100:00100000:2.0:1767318853.596839:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318853.596893:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667904 00000100:00100000:2.0:1767318853.596900:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667904 00000100:00100000:2.0:1767318853.596950:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 499 00000100:00100000:2.0:1767318853.596959:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982667904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318853.597004:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982667904:12345-0@lo:400:kworker.0 Request processed in 46us (164us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318853.597012:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 499 00000100:00080000:2.0:1767318853.597044:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318853.597071:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982667904:0@lo:400:kworker.0 00000100:00100000:3.0:1767318853.597305:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318853.597315:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982667776:0@lo:400:kworker.0 00000100:00100000:3.0:1767318853.597363:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318853.597597:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667776 00000100:00100000:2.0:1767318853.597611:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667776 00000100:00100000:2.0:1767318853.597652:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 500 00000100:00100000:2.0:1767318853.597659:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982667776:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318853.597748:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318853.597887:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982667776:12345-0@lo:400:kworker.0 Request processed in 228us (523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318853.597894:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 500 00000100:00100000:0.0:1767318853.597957:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982667776:0@lo:400:kworker.0 00000100:00080000:1.0:1767318853.605713:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318853.605727:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318853.605733:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982667264:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.605773:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982667520:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.605845:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318853.605919:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982667648:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.605960:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318853.605988:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982668032:0@lo:38:kworker.0 00000100:00100000:1.0:1767318853.606009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318853.606380:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667520 00000100:00100000:3.0:1767318853.606388:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667520 00000100:00100000:3.0:1767318853.606422:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668032 00000100:00100000:3.0:1767318853.606425:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668032 00000100:00100000:3.0:1767318853.606446:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000 00000100:00100000:3.0:1767318853.606454:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982667520:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318853.606473:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318853.606516:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982667520:12345-0@lo:38:kworker.0 Request processed in 62us (670us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318853.606523:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000 00000100:00100000:3.0:1767318853.606537:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001 00000100:00100000:3.0:1767318853.606541:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982668032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318853.606550:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318853.606571:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982668032:12345-0@lo:38:kworker.0 Request processed in 29us (561us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318853.606576:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001 00000100:00080000:3.0:1767318853.606619:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318853.606624:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318853.606627:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982667520:0@lo:38:kworker.0 00000100:00080000:3.0:1767318853.606636:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318853.606639:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318853.606642:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982668032:0@lo:38:kworker.0 00000100:00100000:2.0:1767318853.606972:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982667648 00000100:00100000:2.0:1767318853.606977:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982667648 00000100:00100000:2.0:1767318853.607020:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 275 00000100:00100000:2.0:1767318853.607026:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982667648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318853.607037:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318853.607075:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982667648:12345-0@lo:38:kworker.0 Request processed in 48us (1128us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318853.607081:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 275 00000100:00080000:3.0:1767318853.608003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318853.608009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318853.608013:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982667648:0@lo:38:kworker.0 00010000:00010000:1.0:1767318853.765575:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318854.788514:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318855.110116:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318855.110140:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982668160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318855.110205:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318855.110342:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668160 00000100:00100000:0.0:1767318855.110349:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668160 00000100:00100000:0.0:1767318855.110416:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 530 00000100:00100000:0.0:1767318855.110427:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982668160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318855.110469:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318855.110512:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982668160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (306us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318855.110519:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 530 00000100:00080000:1.0:1767318855.110588:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318855.110842:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982668288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318855.110873:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318855.111086:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668288 00000100:00100000:1.0:1767318855.111091:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668288 00000100:00100000:1.0:1767318855.111130:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 531 00000100:00100000:1.0:1767318855.111138:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982668288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318855.111154:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318855.111190:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982668288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318855.111196:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 531 00000100:00100000:1.0:1767318855.111241:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982668288:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318855.111329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318855.111851:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982668160:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318855.564262:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318855.564339:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592192 00000100:00100000:0.0:1767318855.564352:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592192 00000100:00100000:2.0:1767318855.564398:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318855.564485:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592064 00000100:00100000:0.0:1767318855.564489:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592064 00000100:00080000:1.0:1767318855.564579:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318855.568191:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 766 00000100:00100000:0.0:1767318855.568223:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915592192:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318855.568326:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915592192:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 122us (4075us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318855.568330:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1002 00000100:00100000:0.0:1767318855.568343:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 766 00000100:00100000:3.0:1767318855.568345:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915592064:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318855.568367:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318855.568412:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915592064:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 71us (4013us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318855.568420:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1002 00010000:00010000:1.0:1767318855.812908:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318856.837102:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318857.860495:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318858.692560:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.692571:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318858.692665:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982668416:0@lo:400:kworker.0 00000100:00100000:2.0:1767318858.692736:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318858.692843:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668416 00000100:00100000:1.0:1767318858.692851:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668416 00000100:00100000:1.0:1767318858.692907:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 767 00000100:00100000:1.0:1767318858.692916:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982668416:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767318858.692977:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.692981:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.692985:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.692990:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.692994:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.693000:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318858.693019:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163982668544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318858.693050:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.693053:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.693055:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.693058:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.693061:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.693065:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318858.693075:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0380 x1853163982668800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318858.693084:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.693087:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.693089:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.693092:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.693094:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.693098:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318858.693111:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0700 x1853163982668928/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318858.694702:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318858.694743:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982668544:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.694836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.694903:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982668800:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.694928:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.694946:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982668928:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.694971:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.695021:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668544 00000100:00100000:0.0:1767318858.695030:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668544 00000100:00100000:0.0:1767318858.695066:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668800 00000100:00100000:0.0:1767318858.695069:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668800 00000100:00100000:0.0:1767318858.695085:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1003 00000100:00100000:0.0:1767318858.695093:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982668544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318858.695122:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318858.695172:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982668544:12345-0@lo:38:kworker.0 Request processed in 78us (335us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318858.695179:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1003 00000100:00100000:0.0:1767318858.695194:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004 00000100:00100000:0.0:1767318858.695198:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982668800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318858.695227:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318858.695250:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982668800:12345-0@lo:38:kworker.0 Request processed in 50us (322us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318858.695256:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004 00000100:00100000:0.0:1767318858.695324:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668928 00000100:00100000:0.0:1767318858.695329:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668928 00000100:00100000:0.0:1767318858.695358:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 276 00000100:00100000:0.0:1767318858.695363:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982668928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318858.695372:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318858.695399:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982668928:12345-0@lo:38:kworker.0 Request processed in 35us (427us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318858.695405:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 276 00000100:00080000:0.0:1767318858.695445:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318858.695450:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318858.695454:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982668544:0@lo:38:kworker.0 00000100:00080000:0.0:1767318858.695464:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318858.695467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318858.695470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982668800:0@lo:38:kworker.0 00000100:00080000:0.0:1767318858.695477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318858.695480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318858.695482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982668928:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.695539:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982669184:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695568:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.695620:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982669056:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695642:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.695664:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318858.695670:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982668672:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695690:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318858.695712:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982668672 00000100:00100000:0.0:1767318858.695717:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982668672 00000100:00100000:0.0:1767318858.695752:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 768 00000100:00100000:0.0:1767318858.695758:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982668672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695799:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982668672:12345-0@lo:400:kworker.0 Request processed in 41us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318858.695804:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 768 00000100:00080000:0.0:1767318858.695820:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318858.695838:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1f80 x1853163982668672/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318874 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318858.695851:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982668672:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695880:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669184 00000100:00100000:0.0:1767318858.695884:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669184 00000100:00100000:0.0:1767318858.695919:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669056 00000100:00100000:0.0:1767318858.695922:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669056 00000100:00100000:0.0:1767318858.695941:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 501 00000100:00100000:0.0:1767318858.695946:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982669184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.695980:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982669184:12345-0@lo:400:kworker.0 Request processed in 33us (411us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318858.695987:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 501 00000100:00100000:0.0:1767318858.695997:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 502 00000100:00100000:0.0:1767318858.696008:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982669056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.696037:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982669056:12345-0@lo:400:kworker.0 Request processed in 35us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318858.696043:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 502 00000100:00080000:0.0:1767318858.696061:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318858.696063:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318858.696078:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982669184:0@lo:400:kworker.0 00000100:00100000:0.0:1767318858.696098:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982669056:0@lo:400:kworker.0 00000100:00100000:1.0:1767318858.698648:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982668416:12345-0@lo:400:kworker.0 Request processed in 5723us (5901us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318858.698661:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 767 00010000:00080000:2.0:1767318858.699703:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1500 x1853163982668416/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318874 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318858.699721:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982668416:0@lo:400:kworker.0 00000100:00080000:3.0:1767318858.699820:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318858.699826:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318858.699830:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318858.699835:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4102 00000100:00080000:3.0:1767318858.699839:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318858.699844:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318858.699878:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982669312/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318858.699939:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982669312:0@lo:38:kworker.0 00000100:00100000:0.0:1767318858.699988:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318858.700530:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669312 00000100:00100000:1.0:1767318858.700538:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669312 00000100:00100000:1.0:1767318858.700578:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005 00000100:00100000:1.0:1767318858.700585:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982669312:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318858.700603:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318858.702333:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982669312:12345-0@lo:38:kworker.0 Request processed in 1735us (2331us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318858.702347:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005 00000100:00080000:0.0:1767318858.707890:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318858.707903:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318858.707934:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982669312:0@lo:38:kworker.0 00010000:00010000:1.0:1767318858.889994:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318859.908612:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318860.230969:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318860.230990:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982669440:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318860.231051:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318860.231103:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669440 00000100:00100000:2.0:1767318860.231112:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669440 00000100:00100000:2.0:1767318860.231184:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 532 00000100:00100000:2.0:1767318860.231195:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982669440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318860.231226:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318860.231294:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982669440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (243us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318860.231302:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 532 00000100:00080000:0.0:1767318860.231456:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318860.231599:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982669568:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318860.231639:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318860.231762:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669568 00000100:00100000:3.0:1767318860.231767:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669568 00000100:00100000:3.0:1767318860.231808:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 533 00000100:00100000:3.0:1767318860.231815:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982669568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318860.231832:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318860.231863:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982669568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318860.231869:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 533 00000100:00080000:0.0:1767318860.231919:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318860.231952:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982669568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318860.232472:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982669440:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318860.691060:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767318860.691098:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915592320 offset 0 length 520: 4 00000100:00100000:2.0:1767318860.691125:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318860.691303:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592576 00000100:00100000:1.0:1767318860.691314:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592576 00000100:00100000:1.0:1767318860.691385:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592448 00000100:00100000:1.0:1767318860.691389:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592448 00000100:00100000:1.0:1767318860.691409:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 769 00000100:00100000:1.0:1767318860.691417:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915592576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318860.691494:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915592576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318860.691502:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 769 00000100:00100000:1.0:1767318860.691526:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1006 00000100:00100000:1.0:1767318860.691530:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592448:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318860.691549:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318860.691577:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (451us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318860.691583:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1006 00000100:00080000:2.0:1767318860.691729:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318860.932591:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318861.956658:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318862.981633:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318863.812388:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318863.812402:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318863.812484:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318863.812490:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767318863.812491:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982669696:0@lo:400:kworker.0 00000100:00080000:3.0:1767318863.812494:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318863.812502:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4107 00000100:00080000:3.0:1767318863.812506:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318863.812522:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318863.812545:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163982669824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318863.812569:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318863.812605:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318863.812609:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318863.812612:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318863.812616:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4107 00000100:00080000:3.0:1767318863.812619:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318863.812623:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318863.812636:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901f80 x1853163982670080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318863.812646:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318863.812649:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767318863.812651:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669696 00000100:00080000:3.0:1767318863.812652:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318863.812657:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4107 00000100:00100000:0.0:1767318863.812660:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669696 00000100:00080000:3.0:1767318863.812661:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318863.812666:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318863.812679:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900380 x1853163982670208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318863.812710:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318863.812713:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318863.812715:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767318863.812720:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 770 00000100:00080000:3.0:1767318863.812722:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4107 00000100:00080000:3.0:1767318863.812726:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318863.812732:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318863.812733:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982669696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318863.812745:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901c00 x1853163982670592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318863.812782:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982669696:12345-0@lo:400:kworker.0 Request processed in 53us (214us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318863.812789:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 770 00010000:00080000:3.0:1767318863.812800:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2a00 x1853163982669696/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318879 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318863.812812:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982669696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318863.812827:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318863.812834:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982670464:0@lo:400:kworker.0 00000100:00100000:3.0:1767318863.812862:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318863.812987:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318838/real 1767318838] req@ffff93463a8e0e00 x1853163982663168/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318863 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318863.813006:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e0e00 x1853163982663168/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318863 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318863.813020:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982669824:0@lo:38:kworker.0 00000100:00100000:2.0:1767318863.813046:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318863.813067:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982670080:0@lo:38:kworker.0 00000100:00100000:2.0:1767318863.813083:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318863.813096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982670208:0@lo:38:kworker.0 00000100:00100000:2.0:1767318863.813114:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318863.813131:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982670592:0@lo:38:kworker.0 00000100:00100000:2.0:1767318863.813147:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318863.813162:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318863.813171:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767318863.813174:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982663168:0@lo:250:kworker.0 00000100:00100000:2.0:1767318863.813195:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982670336:0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.813217:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318863.813235:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318863.813241:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982669952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.813260:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318863.813300:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318863.813311:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669824 00000100:00100000:2.0:1767318863.813315:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669824 00000100:00100000:2.0:1767318863.813333:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670080 00000100:00100000:2.0:1767318863.813335:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670080 00000100:00100000:2.0:1767318863.813342:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670592 00000100:00100000:2.0:1767318863.813344:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670592 00000100:00100000:2.0:1767318863.813351:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982669952 00000100:00100000:2.0:1767318863.813353:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982669952 00000100:00100000:2.0:1767318863.813375:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 771 00000100:00100000:2.0:1767318863.813381:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982669952:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.813427:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982669952:12345-0@lo:400:kworker.0 Request processed in 46us (166us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318863.813433:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 771 00000100:00100000:2.0:1767318863.813446:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007 00000100:00100000:2.0:1767318863.813450:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982669824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318863.813476:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318863.813508:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982669824:12345-0@lo:38:kworker.0 Request processed in 57us (462us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318863.813514:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007 00000100:00100000:2.0:1767318863.813524:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008 00000100:00100000:2.0:1767318863.813528:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982670080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318863.813536:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318863.813558:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982670080:12345-0@lo:38:kworker.0 Request processed in 29us (474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318863.813564:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008 00000100:00100000:2.0:1767318863.813574:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1009 00000100:00100000:2.0:1767318863.813577:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982670592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318863.813585:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318863.813605:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982670592:12345-0@lo:38:kworker.0 Request processed in 27us (457us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318863.813610:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1009 00000100:00100000:2.0:1767318863.813626:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670464 00000100:00100000:2.0:1767318863.813629:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670464 00000100:00100000:2.0:1767318863.813651:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670336 00000100:00100000:2.0:1767318863.813654:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670336 00000100:00100000:2.0:1767318863.813670:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 503 00000100:00100000:2.0:1767318863.813675:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982670464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.813706:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982670464:12345-0@lo:400:kworker.0 Request processed in 31us (843us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318863.813712:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 503 00000100:00100000:2.0:1767318863.813723:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 504 00000100:00100000:2.0:1767318863.813729:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982670336:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.813755:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982670336:12345-0@lo:400:kworker.0 Request processed in 27us (536us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318863.813759:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 504 00000100:00080000:2.0:1767318863.813771:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318863.813773:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318863.813775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318863.813790:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318863.813793:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318863.813796:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982669824:0@lo:38:kworker.0 00000100:00080000:2.0:1767318863.813804:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318863.813807:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318863.813809:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982670080:0@lo:38:kworker.0 00000100:00080000:2.0:1767318863.813816:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318863.813819:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318863.813822:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982670592:0@lo:38:kworker.0 00010000:00080000:2.0:1767318863.814003:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631903100 x1853163982669952/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318879 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318863.814019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982669952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318863.814033:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982670336:0@lo:400:kworker.0 00000100:00080000:2.0:1767318863.814059:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318863.814063:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318863.814066:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318863.814071:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4086 00000100:00080000:2.0:1767318863.814074:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318863.814079:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318863.814096:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435a680 x1853163982670720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318863.814187:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670208 00000100:00100000:0.0:1767318863.814192:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670208 00000100:00100000:0.0:1767318863.814221:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 277 00000100:00100000:0.0:1767318863.814228:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982670208:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767318863.814235:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982670720:0@lo:250:kworker.0 00010000:00020000:0.0:1767318863.814240:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000400:00000100:1.0:1767318863.814299:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982670720 offset 0 length 520: 4 00000100:00100000:3.0:1767318863.814370:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982670464:0@lo:400:kworker.0 00000100:00100000:0.0:1767318863.824751:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982670208:12345-0@lo:38:kworker.0 Request processed in 10509us (11621us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318863.824768:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 277 00000100:00080000:1.0:1767318863.828057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318863.828072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318863.828089:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982670208:0@lo:38:kworker.0 00010000:00010000:1.0:1767318864.005464:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318865.028928:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318865.348497:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982670848:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318865.348571:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318865.348651:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670848 00000100:00100000:2.0:1767318865.348659:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670848 00000100:00100000:2.0:1767318865.348732:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 534 00000100:00100000:2.0:1767318865.348742:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982670848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318865.348770:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318865.348817:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982670848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (245us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318865.348824:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 534 00000100:00100000:0.0:1767318865.348981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982670848:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318865.349004:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318865.349181:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318865.349193:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982670976:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318865.349222:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318865.349444:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982670976 00000100:00100000:0.0:1767318865.349450:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982670976 00000100:00080000:3.0:1767318865.349553:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318865.349605:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 535 00000100:00100000:0.0:1767318865.349615:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982670976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318865.349635:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318865.349682:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982670976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (458us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318865.349689:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 535 00000100:00100000:3.0:1767318865.349756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982670976:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318865.803732:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318865.803786:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318865.805586:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592832 00000100:00100000:1.0:1767318865.805599:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592832 00000100:00100000:1.0:1767318865.805687:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592704 00000100:00100000:1.0:1767318865.805691:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592704 00000100:00100000:1.0:1767318865.805714:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 772 00000100:00100000:0.0:1767318865.805723:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1010 00000100:00100000:1.0:1767318865.805724:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915592832:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318865.805728:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592704:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318865.805748:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318865.805802:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915592832:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (2081us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318865.805810:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 772 00000100:00080000:2.0:1767318865.805831:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318865.809543:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592704:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3800us (5744us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318865.809557:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1010 00010000:00010000:1.0:1767318866.052550:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318867.078479:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318868.100575:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318868.932452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318868.932463:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318868.932524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318868.932528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318868.932531:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318868.932539:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4112 00000100:00080000:3.0:1767318868.932544:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318868.932550:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318868.932575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903b80 x1853163982671232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318868.932691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318868.932695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318868.932697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318868.932701:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4112 00000100:00080000:3.0:1767318868.932704:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318868.932708:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318868.932720:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982671488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318868.932728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318868.932731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318868.932733:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318868.932736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4112 00000100:00080000:3.0:1767318868.932738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318868.932741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318868.932751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901880 x1853163982671616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318868.932771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318868.932774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318868.932776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318868.932779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4112 00000100:00080000:3.0:1767318868.932781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318868.932785:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318868.932795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901500 x1853163982672000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318868.932824:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982671360:0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.932893:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982671872:0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.932895:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318868.932938:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318868.932939:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318868.932948:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982671744:0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.932965:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318868.932974:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982671104:0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.932983:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318868.933009:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318868.933021:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671872 00000100:00100000:3.0:1767318868.933029:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671872 00000100:00100000:1.0:1767318868.933071:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671360 00000100:00100000:1.0:1767318868.933076:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671360 00000100:00100000:3.0:1767318868.933087:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671744 00000100:00100000:3.0:1767318868.933092:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671744 00000100:00100000:1.0:1767318868.933110:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671104 00000100:00100000:1.0:1767318868.933114:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671104 00000100:00100000:3.0:1767318868.933124:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 505 00000100:00100000:3.0:1767318868.933135:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982671872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.933148:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 773 00000100:00100000:1.0:1767318868.933157:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982671360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.933202:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982671872:12345-0@lo:400:kworker.0 Request processed in 69us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318868.933210:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982671360:12345-0@lo:400:kworker.0 Request processed in 52us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318868.933214:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 505 00000100:00100000:1.0:1767318868.933223:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 773 00000100:00100000:3.0:1767318868.933237:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 506 00000100:00100000:3.0:1767318868.933245:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982671744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318868.933258:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 774 00000100:00100000:1.0:1767318868.933267:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982671104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.933307:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982671744:12345-0@lo:400:kworker.0 Request processed in 62us (328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318868.933327:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 506 00000100:00100000:1.0:1767318868.933342:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982671104:12345-0@lo:400:kworker.0 Request processed in 76us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318868.933350:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 774 00010000:00080000:3.0:1767318868.933390:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900a80 x1853163982671104/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318884 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318868.933402:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982671104:0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.933413:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982671872:0@lo:400:kworker.0 00000100:00100000:3.0:1767318868.933443:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982671744:0@lo:400:kworker.0 00010000:00080000:3.0:1767318868.933452:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901180 x1853163982671360/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318884 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318868.933459:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982671360:0@lo:400:kworker.0 00000100:00080000:2.0:1767318868.933534:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318868.933540:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318868.933543:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318868.937137:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982671232:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.937227:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318868.937500:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982671488:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.937530:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318868.947964:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982671616:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.948028:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318868.948048:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982672000:0@lo:38:kworker.0 00000100:00100000:0.0:1767318868.948071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318868.948146:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671616 00000100:00100000:3.0:1767318868.948157:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671616 00000100:00100000:3.0:1767318868.948203:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 278 00000100:00100000:3.0:1767318868.948211:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982671616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318868.948233:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318868.948294:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982671616:12345-0@lo:38:kworker.0 Request processed in 82us (265us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318868.948301:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 278 00000100:00100000:2.0:1767318868.948503:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671232 00000100:00100000:2.0:1767318868.948508:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671232 00000100:00100000:2.0:1767318868.948533:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982671488 00000100:00100000:2.0:1767318868.948536:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982671488 00000100:00100000:2.0:1767318868.948548:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011 00000100:00100000:2.0:1767318868.948553:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982671232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318868.948563:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318868.948592:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982671232:12345-0@lo:38:kworker.0 Request processed in 39us (11366us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318868.948597:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011 00000100:00100000:2.0:1767318868.948608:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1012 00000100:00100000:2.0:1767318868.948612:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982671488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318868.948620:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318868.948638:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982671488:12345-0@lo:38:kworker.0 Request processed in 25us (11107us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318868.948643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1012 00000100:00080000:0.0:1767318868.949389:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318868.949398:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318868.949402:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982671232:0@lo:38:kworker.0 00000100:00080000:0.0:1767318868.949412:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318868.949415:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318868.949418:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982671488:0@lo:38:kworker.0 00000100:00080000:0.0:1767318868.949425:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318868.949429:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318868.949431:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982671616:0@lo:38:kworker.0 00000100:00100000:1.0:1767318868.949499:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672000 00000100:00100000:1.0:1767318868.949505:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672000 00000100:00100000:3.0:1767318868.958452:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013 00000100:00100000:3.0:1767318868.958466:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982672000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318868.958491:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318868.958590:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982672000:12345-0@lo:38:kworker.0 Request processed in 125us (10517us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318868.958598:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013 00000100:00080000:0.0:1767318868.960557:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318868.960570:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318868.960574:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982672000:0@lo:38:kworker.0 00010000:00010000:1.0:1767318869.124512:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318870.149513:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318870.470631:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982672128:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318870.470714:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318870.473325:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672128 00000100:00100000:0.0:1767318870.473343:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672128 00000100:00100000:0.0:1767318870.473443:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 536 00000100:00100000:0.0:1767318870.473455:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982672128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318870.473502:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767318870.473506:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318870.473610:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982672128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 158us (2895us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318870.473618:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 536 00000100:00100000:3.0:1767318870.473654:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982672256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318870.473697:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318870.473813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982672128:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318870.477913:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672256 00000100:00100000:0.0:1767318870.477935:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672256 00000100:00100000:0.0:1767318870.478028:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 537 00000100:00100000:0.0:1767318870.478040:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982672256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318870.478080:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318870.478178:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982672256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 140us (4479us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318870.478187:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 537 00000100:00100000:0.0:1767318870.478244:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982672256:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318870.479910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318870.936595:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318870.937003:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318870.937155:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593088 00000100:00100000:3.0:1767318870.937166:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593088 00000100:00100000:0.0:1767318870.937229:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915592960 00000100:00100000:0.0:1767318870.937236:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915592960 00000100:00100000:3.0:1767318870.937256:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 775 00000100:00100000:0.0:1767318870.937266:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1014 00000100:00100000:3.0:1767318870.937309:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915593088:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318870.937323:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592960:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318870.937344:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767318870.937468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318870.937563:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915593088:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 297us (982us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318870.937572:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 775 00000100:00100000:0.0:1767318870.944747:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915592960:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 7411us (7733us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318870.944763:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1014 00010000:00010000:1.0:1767318871.172736:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318872.196502:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318873.226029:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318874.052399:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318874.052410:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318874.052773:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767318874.052799:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318874.052803:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318874.052807:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318874.052818:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4117 00000100:00080000:3.0:1767318874.052823:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318874.052830:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318874.052857:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902300 x1853163982672512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318874.052908:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318874.052912:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318874.052913:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318874.052917:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4117 00000100:00080000:3.0:1767318874.052919:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318874.052923:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318874.052935:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901880 x1853163982672768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318874.052944:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318874.052948:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318874.052949:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318874.052953:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4117 00000100:00080000:3.0:1767318874.052955:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318874.052958:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318874.052970:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982672896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318874.052975:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982672512:0@lo:38:kworker.0 00000100:00100000:2.0:1767318874.053043:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318874.053051:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318874.053061:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982673152:0@lo:400:kworker.0 00000100:00100000:2.0:1767318874.053088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982672768:0@lo:38:kworker.0 00000100:00100000:0.0:1767318874.053092:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318874.053111:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318874.053129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982672896:0@lo:38:kworker.0 00000100:00100000:0.0:1767318874.053140:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672512 00000100:00100000:0.0:1767318874.053147:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672512 00000100:00100000:2.0:1767318874.053150:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318874.053175:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1015 00000100:00100000:0.0:1767318874.053182:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982672512:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767318874.053183:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982673024:0@lo:400:kworker.0 00010000:00020000:0.0:1767318874.053202:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318874.053204:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318874.053234:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318874.053240:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982672640:0@lo:400:kworker.0 00000100:00100000:2.0:1767318874.053258:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318874.053260:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982672512:12345-0@lo:38:kworker.0 Request processed in 77us (217us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318874.053267:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1015 00000100:00100000:0.0:1767318874.053296:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672768 00000100:00100000:0.0:1767318874.053300:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672768 00000100:00100000:2.0:1767318874.053309:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673152 00000100:00100000:0.0:1767318874.053315:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672640 00000100:00100000:0.0:1767318874.053317:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672640 00000100:00100000:0.0:1767318874.053349:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 776 00000100:00100000:0.0:1767318874.053356:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982672640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318874.053395:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982672640:12345-0@lo:400:kworker.0 Request processed in 39us (136us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318874.053401:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 776 00000100:00100000:0.0:1767318874.053414:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016 00000100:00100000:0.0:1767318874.053418:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982672768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318874.053428:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318874.053450:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982672768:12345-0@lo:38:kworker.0 Request processed in 30us (338us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318874.053455:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016 00000100:00100000:2.0:1767318874.053490:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673152 00000100:00100000:0.0:1767318874.053502:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672896 00000100:00100000:0.0:1767318874.053505:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672896 00000100:00100000:2.0:1767318874.053522:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673024 00000100:00100000:0.0:1767318874.053524:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 279 00000100:00100000:2.0:1767318874.053525:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673024 00000100:00100000:0.0:1767318874.053528:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982672896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318874.053538:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318874.053543:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 507 00000100:00100000:2.0:1767318874.053548:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982673152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318874.053561:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982672896:12345-0@lo:38:kworker.0 Request processed in 32us (409us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318874.053567:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 279 00000100:00100000:2.0:1767318874.053579:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982673152:12345-0@lo:400:kworker.0 Request processed in 31us (486us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318874.053584:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 507 00000100:00100000:2.0:1767318874.053596:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 508 00000100:00100000:2.0:1767318874.053602:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982673024:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318874.053608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318874.053611:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318874.053613:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318874.053633:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982673024:12345-0@lo:400:kworker.0 Request processed in 32us (428us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318874.053639:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982673152:0@lo:400:kworker.0 00000100:00100000:2.0:1767318874.053639:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 508 00010000:00080000:2.0:1767318874.053660:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901500 x1853163982672640/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318890 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318874.053670:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982672640:0@lo:400:kworker.0 00000100:00100000:2.0:1767318874.053680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982673024:0@lo:400:kworker.0 00000100:00080000:2.0:1767318874.053700:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318874.053704:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318874.053707:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982672896:0@lo:38:kworker.0 00000100:00080000:2.0:1767318874.054149:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318874.054155:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318874.054158:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982672512:0@lo:38:kworker.0 00000100:00080000:2.0:1767318874.054168:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318874.054172:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318874.054175:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982672768:0@lo:38:kworker.0 00000100:00100000:1.0:1767318874.054519:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982672384:0@lo:400:kworker.0 00000100:00100000:1.0:1767318874.054561:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318874.054651:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982672384 00000100:00100000:0.0:1767318874.054684:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982672384 00000100:00100000:0.0:1767318874.054741:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 777 00000100:00100000:0.0:1767318874.054748:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982672384:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767318874.054818:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318874.054869:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982672384:12345-0@lo:400:kworker.0 Request processed in 121us (306us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318874.054877:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 777 00010000:00080000:1.0:1767318874.054950:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900a80 x1853163982672384/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318890 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318874.054960:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982672384:0@lo:400:kworker.0 00000100:00080000:3.0:1767318874.055117:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318874.055326:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318874.055329:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318874.055334:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4117 00000100:00080000:3.0:1767318874.055337:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318874.055341:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318874.055365:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900380 x1853163982673280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318874.055423:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982673280:0@lo:38:kworker.0 00000100:00100000:2.0:1767318874.055454:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318874.055547:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673280 00000100:00100000:1.0:1767318874.055552:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673280 00000100:00100000:1.0:1767318874.055583:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017 00000100:00100000:1.0:1767318874.055589:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982673280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318874.055609:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318874.055716:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982673280:12345-0@lo:38:kworker.0 Request processed in 125us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318874.055722:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017 00000100:00080000:2.0:1767318874.055768:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318874.055772:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318874.055775:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982673280:0@lo:38:kworker.0 00010000:00010000:1.0:1767318874.244612:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318875.268726:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318875.589890:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318875.589919:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982673408:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318875.589993:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318875.593918:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318875.593941:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982673536:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318875.594011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318875.594633:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673408 00000100:00100000:1.0:1767318875.594643:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673408 00000100:00100000:1.0:1767318875.594701:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673536 00000100:00100000:1.0:1767318875.594704:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673536 00000100:00100000:1.0:1767318875.594722:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 538 00000100:00100000:1.0:1767318875.594732:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982673408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318875.594762:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318875.594815:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982673408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (4821us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318875.594822:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 538 00000100:00100000:1.0:1767318875.594839:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 539 00000100:00100000:1.0:1767318875.594846:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982673536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318875.594860:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318875.594888:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982673536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (876us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318875.594897:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 539 00000100:00100000:1.0:1767318875.594944:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982673408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318875.595045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982673536:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318876.050444:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318876.050501:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318876.050587:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593344 00000100:00100000:0.0:1767318876.050607:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593344 00000100:00100000:0.0:1767318876.050690:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593216 00000100:00100000:0.0:1767318876.050693:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593216 00000100:00100000:0.0:1767318876.050716:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 778 00000100:00100000:0.0:1767318876.050728:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915593344:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767318876.050743:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318876.050794:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1018 00000100:00100000:2.0:1767318876.050802:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915593216:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:0.0:1767318876.050810:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915593344:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318876.050818:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 778 00010000:00020000:2.0:1767318876.050827:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318876.050863:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915593216:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (361us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318876.050871:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1018 00010000:00010000:1.0:1767318876.293667:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318877.316642:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318878.340659:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318879.178788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318879.178799:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318879.178931:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982673664:0@lo:400:kworker.0 00000100:00100000:2.0:1767318879.179006:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318879.179314:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673664 00000100:00100000:3.0:1767318879.179323:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673664 00000100:00100000:3.0:1767318879.179383:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 779 00000100:00100000:3.0:1767318879.179393:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982673664:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318879.179475:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318879.180844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318879.180852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318879.180856:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318879.180862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4122 00000100:00080000:0.0:1767318879.180867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318879.180874:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318879.180907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55500 x1853163982673792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318879.180955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318879.180958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318879.180960:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318879.180963:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4122 00000100:00080000:0.0:1767318879.180965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318879.180968:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318879.180979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982674048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318879.180987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318879.180990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318879.180991:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318879.180994:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4122 00000100:00080000:0.0:1767318879.180997:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318879.181000:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318879.181010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9880 x1853163982674176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318879.181102:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982674432:0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181192:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982673920:0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181323:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181343:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318879.181349:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982674304:0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181369:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181396:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674432 00000100:00100000:1.0:1767318879.181402:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674432 00000100:00100000:1.0:1767318879.181447:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674304 00000100:00100000:1.0:1767318879.181450:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674304 00000100:00100000:1.0:1767318879.181470:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 509 00000100:00100000:1.0:1767318879.181479:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982674432:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181524:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982674432:12345-0@lo:400:kworker.0 Request processed in 45us (331us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318879.181530:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 509 00000100:00100000:1.0:1767318879.181548:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 510 00000100:00100000:1.0:1767318879.181553:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982674304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181582:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982674304:12345-0@lo:400:kworker.0 Request processed in 29us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318879.181587:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 510 00000100:00100000:1.0:1767318879.181604:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673920 00000100:00100000:1.0:1767318879.181607:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673920 00000100:00100000:1.0:1767318879.181633:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 780 00000100:00100000:1.0:1767318879.181639:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982673920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181679:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982673920:12345-0@lo:400:kworker.0 Request processed in 40us (355us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318879.181685:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 780 00000100:00080000:1.0:1767318879.181704:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767318879.181707:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318879.181709:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318879.181736:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982674304:0@lo:400:kworker.0 00010000:00080000:1.0:1767318879.181746:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e56a00 x1853163982673920/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318895 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318879.181755:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982673920:0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181781:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982674432:0@lo:400:kworker.0 00000100:00100000:1.0:1767318879.181846:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982673792:0@lo:38:kworker.0 00000100:00100000:1.0:1767318879.181872:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181894:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982674048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318879.181912:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181925:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982674176:0@lo:38:kworker.0 00000100:00100000:1.0:1767318879.181944:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318879.181972:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982673792 00000100:00100000:1.0:1767318879.181975:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982673792 00000100:00100000:1.0:1767318879.181990:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674048 00000100:00100000:1.0:1767318879.181993:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674048 00000100:00100000:1.0:1767318879.182004:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019 00000100:00100000:1.0:1767318879.182008:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982673792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318879.182025:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318879.182056:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982673792:12345-0@lo:38:kworker.0 Request processed in 46us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318879.182062:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019 00000100:00100000:1.0:1767318879.182071:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020 00000100:00100000:1.0:1767318879.182075:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982674048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318879.182083:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318879.182102:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982674048:12345-0@lo:38:kworker.0 Request processed in 26us (189us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318879.182107:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020 00000100:00080000:1.0:1767318879.182144:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318879.182149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318879.182152:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982673792:0@lo:38:kworker.0 00000100:00080000:1.0:1767318879.182159:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318879.182162:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318879.182164:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982674048:0@lo:38:kworker.0 00000100:00100000:1.0:1767318879.182193:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674176 00000100:00100000:1.0:1767318879.182196:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674176 00000100:00100000:1.0:1767318879.182214:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 280 00000100:00100000:1.0:1767318879.182218:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982674176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318879.182227:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318879.182252:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982674176:12345-0@lo:38:kworker.0 Request processed in 34us (307us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318879.182258:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 280 00000100:00080000:1.0:1767318879.182297:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318879.182302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318879.182304:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982674176:0@lo:38:kworker.0 00000100:00100000:3.0:1767318879.184817:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982673664:12345-0@lo:400:kworker.0 Request processed in 5416us (5801us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318879.184833:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 779 00000100:00080000:0.0:1767318879.191111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318879.191131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318879.191136:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318879.191144:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4122 00000100:00080000:0.0:1767318879.191149:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318879.191156:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318879.191198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163982674560/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318879.191834:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982674560:0@lo:38:kworker.0 00000100:00100000:1.0:1767318879.191907:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767318879.193331:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57480 x1853163982673664/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318895 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318879.193348:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982673664:0@lo:400:kworker.0 00000100:00100000:3.0:1767318879.193538:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674560 00000100:00100000:3.0:1767318879.193547:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674560 00000100:00100000:3.0:1767318879.193609:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1021 00000100:00100000:3.0:1767318879.193617:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982674560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318879.193639:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318879.193888:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982674560:12345-0@lo:38:kworker.0 Request processed in 270us (1981us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318879.193896:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1021 00000100:00080000:1.0:1767318879.193968:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318879.193973:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318879.193976:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982674560:0@lo:38:kworker.0 00010000:00010000:1.0:1767318879.364923:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318880.390549:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318880.708537:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318880.708561:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982674688:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318880.708630:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318880.708866:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318880.708876:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982674816:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318880.708904:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318880.709010:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674688 00000100:00100000:1.0:1767318880.709019:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674688 00000100:00100000:1.0:1767318880.709067:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674816 00000100:00100000:1.0:1767318880.709071:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674816 00000100:00100000:1.0:1767318880.709087:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 540 00000100:00100000:1.0:1767318880.709125:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982674688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318880.709152:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318880.709204:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982674688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (573us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318880.709212:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 540 00000100:00100000:1.0:1767318880.709230:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 541 00000100:00100000:1.0:1767318880.709236:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982674816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318880.709252:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318880.709304:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982674816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (399us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318880.709311:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 541 00000100:00100000:3.0:1767318880.709386:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982674816:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318880.709418:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982674688:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318881.165410:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318881.165466:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318881.165638:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593600 00000100:00100000:3.0:1767318881.165648:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593600 00000100:00100000:3.0:1767318881.165728:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593472 00000100:00100000:3.0:1767318881.165732:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593472 00000100:00100000:3.0:1767318881.165749:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 781 00000100:00100000:3.0:1767318881.165758:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915593600:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318881.165830:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915593600:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (432us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318881.165838:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 781 00000100:00100000:3.0:1767318881.165856:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1022 00000100:00100000:3.0:1767318881.165861:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915593472:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318881.165879:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318881.165903:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915593472:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (437us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318881.165909:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1022 00000100:00080000:2.0:1767318881.166014:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318881.412549:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318882.436515:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318883.460855:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318884.295839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318884.295849:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318884.295941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318884.295945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318884.295950:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318884.295960:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4127 00000100:00080000:3.0:1767318884.295965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318884.295971:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318884.295988:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318884.295994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901c00 x1853163982675072/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318884.296005:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982674944:0@lo:400:kworker.0 00000100:00080000:3.0:1767318884.296046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318884.296049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318884.296051:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318884.296054:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4127 00000100:00080000:3.0:1767318884.296056:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318884.296060:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318884.296072:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900380 x1853163982675328/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318884.296077:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318884.296081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318884.296084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318884.296086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318884.296089:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4127 00000100:00080000:3.0:1767318884.296090:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318884.296093:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318884.296103:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163982675456/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318884.296122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318884.296125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318884.296127:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318884.296131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4127 00000100:00080000:3.0:1767318884.296133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318884.296137:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318884.296142:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982675712:0@lo:400:kworker.0 00000100:00100000:3.0:1767318884.296146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982675840/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318884.296167:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318884.296178:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982675200:0@lo:400:kworker.0 00000100:00100000:1.0:1767318884.296201:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982674944 00000100:00100000:3.0:1767318884.296205:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318884.296209:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982674944 00000100:00100000:3.0:1767318884.296223:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318884.296228:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982675584:0@lo:400:kworker.0 00000100:00100000:3.0:1767318884.296246:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318884.296253:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675200 00000100:00100000:1.0:1767318884.296256:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675200 00000100:00100000:3.0:1767318884.296303:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 782 00000100:00080000:2.0:1767318884.296350:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318884.296365:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982675072:0@lo:38:kworker.0 00000100:00100000:0.0:1767318884.296379:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675712 00000100:00100000:0.0:1767318884.296384:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675712 00000100:00100000:2.0:1767318884.296392:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318884.296414:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982675328:0@lo:38:kworker.0 00000100:00100000:0.0:1767318884.296426:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675584 00000100:00100000:0.0:1767318884.296429:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675584 00000100:00100000:2.0:1767318884.296434:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318884.296449:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982675456:0@lo:38:kworker.0 00000100:00100000:2.0:1767318884.296470:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318884.296490:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982675840:0@lo:38:kworker.0 00000100:00100000:2.0:1767318884.296510:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318884.296536:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675072 00000100:00100000:2.0:1767318884.296539:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675072 00000100:00100000:0.0:1767318884.296582:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 511 00000100:00100000:0.0:1767318884.296591:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982675712:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318884.296651:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982675712:12345-0@lo:400:kworker.0 Request processed in 62us (482us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318884.296657:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 511 00000100:00100000:0.0:1767318884.296672:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 512 00000100:00100000:0.0:1767318884.296677:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982675584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318884.296707:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982675584:12345-0@lo:400:kworker.0 Request processed in 30us (460us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318884.296713:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 512 00000100:00100000:0.0:1767318884.296750:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982675584:0@lo:400:kworker.0 00000100:00100000:0.0:1767318884.296796:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982675712:0@lo:400:kworker.0 00000100:00100000:0.0:1767318884.296831:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675456 00000100:00100000:0.0:1767318884.296835:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675456 00000100:00100000:0.0:1767318884.296856:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 281 00000100:00100000:0.0:1767318884.296860:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982675456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318884.296876:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318884.296907:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982675456:12345-0@lo:38:kworker.0 Request processed in 46us (437us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318884.296912:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 281 00000100:00100000:0.0:1767318884.296928:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675328 00000100:00100000:0.0:1767318884.296931:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675328 00000100:00100000:3.0:1767318884.297613:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982674944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318884.297662:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982674944:12345-0@lo:400:kworker.0 Request processed in 1351us (1584us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318884.303580:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675840 00000100:00100000:1.0:1767318884.303596:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675840 00000100:00100000:1.0:1767318884.303631:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 783 00000100:00100000:1.0:1767318884.303647:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982675200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318884.303766:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982675200:12345-0@lo:400:kworker.0 Request processed in 126us (7558us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318884.303774:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 783 00000100:00100000:1.0:1767318884.303793:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1767318884.303799:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982675840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318884.303819:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318884.303851:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982675840:12345-0@lo:38:kworker.0 Request processed in 51us (7340us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318884.303858:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023 00010000:00080000:1.0:1767318884.303912:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901f80 x1853163982675200/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318900 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318884.303926:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982675200:0@lo:400:kworker.0 00000100:00080000:1.0:1767318884.303997:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318884.304002:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318884.304038:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318884.304043:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318884.304050:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982675456:0@lo:38:kworker.0 00000100:00080000:1.0:1767318884.304063:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318884.304066:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318884.304068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982675840:0@lo:38:kworker.0 00010000:00080000:1.0:1767318884.304111:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631903100 x1853163982674944/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318900 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318884.304121:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982674944:0@lo:400:kworker.0 00000100:00100000:2.0:1767318884.305546:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024 00000100:00100000:2.0:1767318884.305556:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982675072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318884.305576:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318884.305650:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982675072:12345-0@lo:38:kworker.0 Request processed in 94us (9257us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318884.305657:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024 00000100:00080000:2.0:1767318884.305697:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318884.305702:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318884.306167:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982675072:0@lo:38:kworker.0 00000100:00100000:0.0:1767318884.318890:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025 00000100:00100000:0.0:1767318884.318903:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982675328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318884.318927:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318884.319020:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982675328:12345-0@lo:38:kworker.0 Request processed in 117us (22584us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318884.319037:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025 00000100:00080000:1.0:1767318884.319168:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318884.319173:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318884.319178:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982675328:0@lo:38:kworker.0 00000100:00100000:3.0:1767318884.319941:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 782 00010000:00010000:1.0:1767318884.486647:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318885.263741:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767318885.263778:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915593728 offset 0 length 520: 4 00000100:00100000:0.0:1767318885.263802:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318885.263864:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593984 00000100:00100000:3.0:1767318885.263874:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593984 00000100:00100000:2.0:1767318885.263942:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915593856 00000100:00100000:2.0:1767318885.263948:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915593856 00000100:00100000:3.0:1767318885.263981:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 784 00000100:00100000:2.0:1767318885.263993:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1026 00000100:00100000:3.0:1767318885.263993:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915593984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318885.264035:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915593856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318885.264057:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318885.264095:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915593856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (287us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318885.264095:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915593984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (360us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318885.264110:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 784 00000100:00100000:2.0:1767318885.264122:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1026 00000100:00080000:1.0:1767318885.265538:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318885.509455:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318885.829342:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318885.829366:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982675968:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318885.829446:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318885.829523:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982675968 00000100:00100000:3.0:1767318885.829531:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982675968 00000100:00100000:3.0:1767318885.829576:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 542 00000100:00100000:3.0:1767318885.829585:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982675968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318885.829616:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318885.829659:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982675968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (225us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318885.829665:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 542 00000100:00100000:3.0:1767318885.829708:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982675968:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318885.833657:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982676096:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318885.833726:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318885.833788:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676096 00000100:00100000:0.0:1767318885.833795:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676096 00000100:00100000:0.0:1767318885.833838:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 543 00000100:00100000:0.0:1767318885.833846:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982676096:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318885.833876:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318885.833920:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982676096:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (193us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318885.833927:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 543 00000100:00100000:0.0:1767318885.833973:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982676096:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318886.536369:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318887.557103:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318888.388468:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318863/real 1767318863] req@ffff93460435a680 x1853163982670720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318888 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318888.388507:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460435a680 x1853163982670720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318888 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767318888.388526:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318888.388562:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767318888.388566:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982670720:0@lo:250:kworker.0 00000100:00080000:3.0:1767318888.388693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318888.388698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767318888.388700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318888.388706:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4112 00000100:00080000:3.0:1767318888.388709:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318888.388716:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318888.388747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903800 x1853163982676224/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318888.388809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318888.388813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318888.388814:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318888.388818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4132 00000100:00080000:3.0:1767318888.388821:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318888.388824:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318888.388839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903100 x1853163982676480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318888.388866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318888.388870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318888.388872:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318888.388875:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4132 00000100:00080000:3.0:1767318888.388878:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318888.388881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318888.388894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163982676736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318888.388902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318888.388904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318888.388906:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318888.388908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4132 00000100:00080000:3.0:1767318888.388910:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318888.388913:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318888.388923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901f80 x1853163982676864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318888.388942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318888.388945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318888.388977:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318888.388981:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4132 00000100:00080000:3.0:1767318888.388983:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318888.388986:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318888.388997:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982677248/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318888.389024:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982676992:0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.389141:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318888.389330:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318888.389336:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982676608:0@lo:400:kworker.0 00000100:00100000:3.0:1767318888.389364:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.389454:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982676224:0@lo:250:kworker.0 00000400:00000100:1.0:1767318888.389478:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982676224 offset 0 length 520: 4 00000100:00100000:1.0:1767318888.389495:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982676480:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.389516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.389540:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982676736:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.389560:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.389573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982676864:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.389592:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.389609:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982677248:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.389627:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318888.389659:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676864 00000100:00100000:1.0:1767318888.389666:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676864 00000100:00100000:1.0:1767318888.389698:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 282 00000100:00100000:1.0:1767318888.389705:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982676864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318888.389724:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318888.389764:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982676864:12345-0@lo:38:kworker.0 Request processed in 59us (171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318888.389771:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 282 00000100:00080000:1.0:1767318888.389806:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318888.389811:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318888.389814:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982676864:0@lo:38:kworker.0 00000100:00100000:1.0:1767318888.389834:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676608 00000100:00100000:1.0:1767318888.389838:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676608 00000100:00100000:1.0:1767318888.389872:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676480 00000100:00100000:1.0:1767318888.389875:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676480 00000100:00100000:1.0:1767318888.389885:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676736 00000100:00100000:1.0:1767318888.389887:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676736 00000100:00100000:1.0:1767318888.389893:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677248 00000100:00100000:1.0:1767318888.389895:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677248 00000100:00100000:1.0:1767318888.389906:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 785 00000100:00100000:1.0:1767318888.389912:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982676608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318888.389967:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982676608:12345-0@lo:400:kworker.0 Request processed in 56us (602us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318888.389974:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 785 00000100:00100000:1.0:1767318888.389987:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1027 00000100:00100000:1.0:1767318888.389990:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982676480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318888.389999:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318888.390025:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982676480:12345-0@lo:38:kworker.0 Request processed in 33us (508us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318888.390031:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1027 00000100:00100000:1.0:1767318888.390043:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028 00000100:00100000:1.0:1767318888.390047:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982676736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318888.390054:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318888.390067:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676992 00000100:00100000:1.0:1767318888.390074:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982676736:12345-0@lo:38:kworker.0 Request processed in 26us (513us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318888.390075:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676992 00000100:00100000:1.0:1767318888.390083:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028 00000100:00100000:1.0:1767318888.390093:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029 00000100:00100000:1.0:1767318888.390096:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982677248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318888.390102:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318888.390118:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982677248:12345-0@lo:38:kworker.0 Request processed in 21us (489us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318888.390123:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 513 00000100:00100000:1.0:1767318888.390128:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029 00000100:00100000:0.0:1767318888.390137:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982676992:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318888.390146:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318888.390190:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982676992:12345-0@lo:400:kworker.0 Request processed in 58us (1050us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318888.390197:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 513 00000100:00100000:0.0:1767318888.390228:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982676352:0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.390256:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318888.390322:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318888.390328:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982677120:0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.390365:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318888.390408:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677120 00000100:00100000:0.0:1767318888.390412:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677120 00000100:00100000:0.0:1767318888.390437:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 514 00000100:00100000:0.0:1767318888.390443:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982677120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.390477:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982677120:12345-0@lo:400:kworker.0 Request processed in 34us (112us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318888.390483:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 514 00000100:00100000:0.0:1767318888.390500:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982676352 00000100:00100000:0.0:1767318888.390503:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982676352 00000100:00100000:0.0:1767318888.390529:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 786 00000100:00100000:0.0:1767318888.390535:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982676352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.390565:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982676352:12345-0@lo:400:kworker.0 Request processed in 30us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318888.390571:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 786 00000100:00100000:0.0:1767318888.390600:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982677120:0@lo:400:kworker.0 00010000:00080000:0.0:1767318888.390612:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901c00 x1853163982676352/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318904 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318888.390621:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982676352:0@lo:400:kworker.0 00010000:00080000:0.0:1767318888.390669:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631902d80 x1853163982676608/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318904 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318888.390678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982676608:0@lo:400:kworker.0 00000100:00100000:0.0:1767318888.390688:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982676992:0@lo:400:kworker.0 00000100:00080000:0.0:1767318888.390718:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318888.390723:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318888.390726:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982676480:0@lo:38:kworker.0 00000100:00080000:0.0:1767318888.390735:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318888.390738:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318888.390740:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982676736:0@lo:38:kworker.0 00000100:00080000:0.0:1767318888.390749:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318888.390753:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318888.390755:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982677248:0@lo:38:kworker.0 00000100:00080000:1.0:1767318888.391134:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318888.391137:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318888.580629:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318889.609669:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318890.382324:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318890.382427:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594240 00000100:00100000:2.0:1767318890.382450:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594240 00000100:00100000:2.0:1767318890.382515:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 787 00000100:00100000:2.0:1767318890.382525:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915594240:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318890.382614:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915594240:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318890.382623:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 787 00000100:00080000:1.0:1767318890.382680:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318890.382881:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318890.383019:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594112 00000100:00100000:3.0:1767318890.383025:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594112 00000100:00100000:3.0:1767318890.383057:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1030 00000100:00100000:3.0:1767318890.383074:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915594112:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318890.383100:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318890.383143:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915594112:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 69us (262us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318890.383149:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1030 00010000:00010000:2.0:1767318890.628603:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318890.948601:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318890.948625:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982677376:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318890.948697:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318890.948872:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677376 00000100:00100000:3.0:1767318890.948881:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677376 00000100:00100000:3.0:1767318890.948988:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 544 00000100:00100000:3.0:1767318890.948998:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982677376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318890.949027:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318890.949086:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982677376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (389us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318890.949094:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 544 00000100:00100000:3.0:1767318890.949138:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982677376:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318890.949694:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318890.949705:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982677504:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318890.949740:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318890.949947:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677504 00000100:00100000:2.0:1767318890.949954:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677504 00000100:00100000:2.0:1767318890.949993:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 545 00000100:00100000:2.0:1767318890.950002:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982677504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318890.950020:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318890.950056:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982677504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (315us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318890.950062:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 545 00000100:00100000:0.0:1767318890.950179:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982677504:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318890.950254:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318890.950259:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767318891.653153:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318892.676977:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318893.508423:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318893.508434:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767318893.508530:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767318893.508551:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318893.508555:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318893.508558:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318893.508567:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318893.508571:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318893.508578:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318893.508600:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962300 x1853163982677760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318893.508646:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318893.508649:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318893.508651:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318893.508654:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318893.508656:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318893.508660:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318893.508672:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982678016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318893.508681:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318893.508684:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318893.508686:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318893.508688:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318893.508691:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318893.508694:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318893.508727:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163982678144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318893.508800:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318893.508804:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982677760:0@lo:38:kworker.0 00000100:00100000:2.0:1767318893.508808:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982678400:0@lo:400:kworker.0 00000100:00100000:3.0:1767318893.511013:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982677632:0@lo:400:kworker.0 00000100:00080000:0.0:1767318893.511173:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318893.511179:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318893.511182:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318893.511188:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318893.511191:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318893.511197:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318893.511260:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.511703:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982678016:0@lo:38:kworker.0 00000100:00100000:1.0:1767318893.511735:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.511756:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982678144:0@lo:38:kworker.0 00000100:00100000:1.0:1767318893.511782:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.511839:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982678272:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.511880:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318893.511903:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318893.511910:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982677888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.511935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318893.512060:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318893.512101:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677760 00000100:00100000:2.0:1767318893.512109:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677760 00000100:00100000:1.0:1767318893.512137:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678272 00000100:00100000:1.0:1767318893.512143:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678272 00000100:00100000:2.0:1767318893.512144:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678016 00000100:00100000:2.0:1767318893.512149:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678016 00000100:00100000:2.0:1767318893.512168:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677888 00000100:00100000:2.0:1767318893.512171:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677888 00000100:00100000:1.0:1767318893.512197:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678400 00000100:00100000:1.0:1767318893.512201:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678400 00000100:00100000:1.0:1767318893.512226:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 515 00000100:00100000:1.0:1767318893.512235:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982678272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318893.512429:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 788 00000100:00100000:2.0:1767318893.512439:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982677888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318893.512480:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982677888:12345-0@lo:400:kworker.0 Request processed in 44us (544us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318893.512487:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 788 00000100:00100000:2.0:1767318893.512503:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031 00000100:00100000:2.0:1767318893.512507:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982677760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318893.512525:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318893.512558:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982677760:12345-0@lo:38:kworker.0 Request processed in 50us (1300us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318893.512565:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031 00000100:00100000:2.0:1767318893.512576:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032 00000100:00100000:2.0:1767318893.512580:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982678016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318893.512588:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318893.512619:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982678016:12345-0@lo:38:kworker.0 Request processed in 37us (883us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318893.512626:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032 00000100:00100000:1.0:1767318893.512635:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982678272:12345-0@lo:400:kworker.0 Request processed in 398us (751us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318893.512644:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 515 00000100:00100000:1.0:1767318893.512662:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 516 00010000:00080000:2.0:1767318893.512671:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962680 x1853163982677888/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318909 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318893.512672:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982678400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318893.512693:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982677888:0@lo:400:kworker.0 00000100:00100000:2.0:1767318893.512709:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982678272:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.512723:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982678400:12345-0@lo:400:kworker.0 Request processed in 52us (672us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318893.512730:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 516 00000100:00100000:2.0:1767318893.512749:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982678400:0@lo:400:kworker.0 00000100:00100000:1.0:1767318893.512758:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678144 00000100:00100000:1.0:1767318893.512764:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678144 00000100:00100000:1.0:1767318893.512788:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 283 00000100:00100000:1.0:1767318893.512793:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982678144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318893.512805:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767318893.512815:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318893.512821:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767318893.512823:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318893.512840:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982678144:12345-0@lo:38:kworker.0 Request processed in 43us (1054us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767318893.512844:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767318893.512849:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 283 00000100:00080000:2.0:1767318893.512866:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318893.512870:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982677760:0@lo:38:kworker.0 00000100:00080000:2.0:1767318893.512880:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318893.512884:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318893.512886:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982678016:0@lo:38:kworker.0 00000100:00080000:2.0:1767318893.512894:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318893.512897:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318893.512899:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982678144:0@lo:38:kworker.0 00000100:00100000:0.0:1767318893.513040:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597800 x1853163982678528/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318893.513190:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982678528:0@lo:38:kworker.0 00000100:00100000:2.0:1767318893.514402:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318893.518732:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678528 00000100:00100000:0.0:1767318893.518747:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678528 00000100:00100000:0.0:1767318893.518811:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1033 00000100:00100000:0.0:1767318893.518820:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982678528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318893.518843:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318893.518951:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982678528:12345-0@lo:38:kworker.0 Request processed in 131us (4552us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318893.518959:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1033 00000100:00080000:1.0:1767318893.519501:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318893.519508:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318893.519514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982678528:0@lo:38:kworker.0 00000100:00100000:3.0:1767318893.521085:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318893.525185:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982677632 00000100:00100000:2.0:1767318893.525200:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982677632 00000100:00100000:2.0:1767318893.525336:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 789 00000100:00100000:2.0:1767318893.525346:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982677632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318893.525446:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982677632:12345-0@lo:400:kworker.0 Request processed in 102us (4367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318893.525453:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 789 00000100:00080000:1.0:1767318893.525692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767318893.525715:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163982677632/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318909 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318893.525733:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982677632:0@lo:400:kworker.0 00010000:00010000:2.0:1767318893.700726:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318894.735684:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318895.501797:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318895.503145:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318895.507016:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594368 00000100:00100000:2.0:1767318895.507031:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594368 00000100:00100000:2.0:1767318895.507160:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594496 00000100:00100000:2.0:1767318895.507164:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594496 00000100:00100000:2.0:1767318895.507312:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 790 00000100:00100000:2.0:1767318895.507323:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915594496:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318895.507416:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915594496:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (4277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318895.507424:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 790 00000100:00100000:0.0:1767318895.507846:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1034 00000100:00100000:0.0:1767318895.507853:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915594368:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318895.507875:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318895.507921:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915594368:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 67us (6135us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318895.507929:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1034 00000100:00080000:1.0:1767318895.509383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318895.749097:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318896.068775:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982678656:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318896.068863:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318896.069355:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678656 00000100:00100000:3.0:1767318896.069364:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678656 00000100:00100000:3.0:1767318896.069441:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 546 00000100:00100000:3.0:1767318896.069451:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982678656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318896.069483:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318896.069530:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982678656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (667us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318896.069537:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 546 00000100:00080000:1.0:1767318896.069603:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318896.070015:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318896.070027:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982678784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318896.070061:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318896.070195:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678784 00000100:00100000:0.0:1767318896.070201:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678784 00000100:00100000:2.0:1767318896.070484:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982678656:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318896.070597:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 547 00000100:00100000:0.0:1767318896.070606:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982678784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318896.070628:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318896.070680:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982678784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (618us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318896.070687:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 547 00000100:00100000:3.0:1767318896.071460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982678784:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318896.074675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318896.772459:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318897.796454:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318898.629211:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.629221:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318898.629322:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982678912:0@lo:400:kworker.0 00000100:00100000:1.0:1767318898.629395:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318898.629489:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982678912 00000100:00100000:2.0:1767318898.629497:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982678912 00000100:00100000:2.0:1767318898.629559:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 791 00000100:00100000:2.0:1767318898.629568:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982678912:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767318898.629638:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318898.629752:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.629783:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.629787:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.629792:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.629796:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.629801:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318898.629821:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5180 x1853163982679040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318898.629859:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.629862:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.629863:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.629866:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.629869:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.629872:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318898.629881:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5880 x1853163982679296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318898.629890:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.629893:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.629895:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.629897:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.629899:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.629901:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318898.629913:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6300 x1853163982679424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318898.629965:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982679680:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.629998:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630026:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679680 00000100:00100000:0.0:1767318898.630029:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679680 00000100:00100000:0.0:1767318898.630053:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 517 00000100:00100000:0.0:1767318898.630058:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982679680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630110:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982679680:12345-0@lo:400:kworker.0 Request processed in 52us (111us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318898.630118:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 517 00000100:00100000:0.0:1767318898.630159:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982679680:0@lo:400:kworker.0 00000100:00080000:0.0:1767318898.630177:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318898.630187:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982679040:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.630210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982679296:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.630241:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630252:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982679424:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.630289:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630324:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679040 00000100:00100000:0.0:1767318898.630328:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679040 00000100:00100000:0.0:1767318898.630345:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679296 00000100:00100000:0.0:1767318898.630347:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679296 00000100:00100000:0.0:1767318898.630360:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035 00000100:00100000:0.0:1767318898.630364:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982679040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.630380:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.630407:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982679040:12345-0@lo:38:kworker.0 Request processed in 42us (197us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.630413:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035 00000100:00100000:0.0:1767318898.630422:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036 00000100:00100000:0.0:1767318898.630425:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982679296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.630432:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.630447:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982679296:12345-0@lo:38:kworker.0 Request processed in 21us (206us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.630452:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036 00000100:00080000:0.0:1767318898.630477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.630481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.630484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982679040:0@lo:38:kworker.0 00000100:00080000:0.0:1767318898.630490:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.630493:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.630495:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982679296:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.630519:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982679168:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630538:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630552:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767318898.630556:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982679552:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630570:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318898.630585:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679552 00000100:00100000:0.0:1767318898.630588:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679552 00000100:00100000:0.0:1767318898.630607:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 518 00000100:00100000:0.0:1767318898.630613:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982679552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630639:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982679552:12345-0@lo:400:kworker.0 Request processed in 27us (68us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318898.630644:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 518 00000100:00100000:0.0:1767318898.630657:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679168 00000100:00100000:0.0:1767318898.630659:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679168 00000100:00100000:0.0:1767318898.630676:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 792 00000100:00100000:0.0:1767318898.630680:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982679168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630701:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982679168:12345-0@lo:400:kworker.0 Request processed in 21us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318898.630706:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 792 00000100:00080000:0.0:1767318898.630718:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318898.630720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318898.630731:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982679552:0@lo:400:kworker.0 00010000:00080000:0.0:1767318898.630739:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6d80 x1853163982679168/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318914 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318898.630748:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982679168:0@lo:400:kworker.0 00000100:00100000:0.0:1767318898.630784:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679424 00000100:00100000:0.0:1767318898.630788:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679424 00000100:00100000:0.0:1767318898.630805:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 284 00000100:00100000:0.0:1767318898.630809:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982679424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318898.630819:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318898.630846:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982679424:12345-0@lo:38:kworker.0 Request processed in 35us (556us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318898.630852:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 284 00000100:00080000:0.0:1767318898.630871:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318898.630874:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318898.630877:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982679424:0@lo:38:kworker.0 00000100:00100000:2.0:1767318898.632710:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982678912:12345-0@lo:400:kworker.0 Request processed in 3135us (3306us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318898.632722:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 791 00000100:00080000:3.0:1767318898.632830:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318898.632835:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318898.632838:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318898.632843:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4142 00000100:00080000:3.0:1767318898.632847:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318898.632852:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318898.632880:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4700 x1853163982679808/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318898.632971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982679808:0@lo:38:kworker.0 00000100:00100000:0.0:1767318898.633017:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318898.633211:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679808 00000100:00100000:2.0:1767318898.633217:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679808 00000100:00100000:2.0:1767318898.633243:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037 00000100:00100000:2.0:1767318898.633248:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982679808:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318898.633291:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318898.633328:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982679808:12345-0@lo:38:kworker.0 Request processed in 79us (310us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318898.633334:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037 00000100:00080000:3.0:1767318898.633472:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318898.633477:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318898.633481:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982679808:0@lo:38:kworker.0 00010000:00080000:1.0:1767318898.633676:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7b80 x1853163982678912/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318914 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318898.633687:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982678912:0@lo:400:kworker.0 00010000:00010000:1.0:1767318898.820507:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318899.845781:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318900.623938:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318900.623993:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318900.627310:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594752 00000100:00100000:2.0:1767318900.627328:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594752 00000100:00100000:2.0:1767318900.628073:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594624 00000100:00100000:2.0:1767318900.628080:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594624 00000100:00100000:2.0:1767318900.628292:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 793 00000100:00100000:2.0:1767318900.628304:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915594752:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318900.628530:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1038 00000100:00100000:3.0:1767318900.628538:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915594624:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318900.628559:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318900.629031:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915594752:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 727us (5102us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318900.629042:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 793 00000100:00080000:1.0:1767318900.629217:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318900.630512:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915594624:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 1966us (6511us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318900.630524:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1038 00010000:00010000:1.0:1767318900.868702:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318901.188748:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318901.188815:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318901.188830:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982680064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318901.188916:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318901.188993:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680064 00000100:00100000:3.0:1767318901.189001:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680064 00000100:00100000:3.0:1767318901.189062:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 548 00000100:00100000:3.0:1767318901.189073:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982680064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318901.189112:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318901.189172:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982680064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318901.189181:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 548 00000100:00080000:3.0:1767318901.189204:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318901.189242:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982680064:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318901.189991:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982679936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318901.190020:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318901.190217:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982679936 00000100:00100000:1.0:1767318901.190223:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982679936 00000100:00080000:3.0:1767318901.190799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318901.190871:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 549 00000100:00100000:1.0:1767318901.190881:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982679936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318901.190910:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318901.190968:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982679936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (946us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318901.190976:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 549 00000100:00100000:0.0:1767318901.191014:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982679936:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318901.892826:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318902.916569:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767318903.748923:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318903.748936:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767318903.749029:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982680192:0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.749114:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318903.749254:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318903.749259:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318903.749263:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318903.749287:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4147 00000100:00080000:0.0:1767318903.749292:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318903.749298:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318903.749320:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163982680320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318903.749360:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318903.749364:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318903.749366:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318903.749370:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4147 00000100:00100000:2.0:1767318903.749372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982680320:0@lo:38:kworker.0 00000100:00080000:0.0:1767318903.749373:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318903.749376:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318903.749387:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597800 x1853163982680576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318903.749395:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767318903.749400:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318903.749409:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318903.749411:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318903.749417:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4147 00000100:00080000:0.0:1767318903.749420:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318903.749423:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318903.749435:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163982680704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318903.750362:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680192 00000100:00100000:1.0:1767318903.750374:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680192 00000100:00100000:2.0:1767318903.750403:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982680576:0@lo:38:kworker.0 00000100:00080000:0.0:1767318903.750519:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318903.750523:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318903.750526:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318903.750530:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4147 00000100:00080000:0.0:1767318903.750532:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318903.750536:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318903.750553:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318903.750577:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982680704:0@lo:38:kworker.0 00000100:00100000:2.0:1767318903.750606:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318903.750642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982680448:0@lo:400:kworker.0 00000100:00100000:1.0:1767318903.750710:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680320 00000100:00100000:1.0:1767318903.750715:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680320 00000100:00100000:1.0:1767318903.750729:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680576 00000100:00100000:1.0:1767318903.750732:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680576 00000100:00100000:1.0:1767318903.750745:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 794 00000100:00100000:1.0:1767318903.750754:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982680192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.750921:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318903.750929:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982680960:0@lo:400:kworker.0 00000100:00100000:0.0:1767318903.751015:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0700 x1853163982681088/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318903.751052:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1039 00000100:00100000:0.0:1767318903.751057:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982680320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318903.751074:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318903.754025:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318903.755304:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318903.755364:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318903.755400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982680832:0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.755446:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318903.755481:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680960 00000100:00100000:3.0:1767318903.755487:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680960 00000100:00100000:3.0:1767318903.755530:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680832 00000100:00100000:3.0:1767318903.755533:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680832 00000100:00100000:3.0:1767318903.755745:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 519 00000100:00100000:3.0:1767318903.755754:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982680960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.755826:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982680960:12345-0@lo:400:kworker.0 Request processed in 73us (524us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318903.755834:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 519 00000100:00100000:3.0:1767318903.755850:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 520 00000100:00100000:3.0:1767318903.755855:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982680832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.755880:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982680832:12345-0@lo:400:kworker.0 Request processed in 26us (434us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318903.755885:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 520 00000100:00080000:3.0:1767318903.755899:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318903.755901:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318903.755927:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982680832:0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.755951:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982680960:0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.755991:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680448 00000100:00100000:3.0:1767318903.755995:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680448 00000100:00100000:3.0:1767318903.756020:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795 00000100:00100000:3.0:1767318903.756025:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982680448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318903.756057:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982680448:12345-0@lo:400:kworker.0 Request processed in 31us (2039us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318903.756062:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795 00000100:00100000:3.0:1767318903.756073:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040 00000100:00100000:3.0:1767318903.756078:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982680576:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318903.756227:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318903.756258:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982680576:12345-0@lo:38:kworker.0 Request processed in 179us (5705us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318903.756265:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040 00000100:00100000:3.0:1767318903.756308:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982680704 00000100:00100000:3.0:1767318903.756312:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982680704 00000100:00100000:3.0:1767318903.756330:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 285 00000100:00100000:3.0:1767318903.756335:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982680704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318903.756344:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318903.756369:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982680704:12345-0@lo:38:kworker.0 Request processed in 33us (5762us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318903.756375:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 285 00000100:00080000:3.0:1767318903.756389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318903.756473:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982681088:0@lo:38:kworker.0 00000100:00100000:3.0:1767318903.756502:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318903.756528:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318903.756534:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318903.756538:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982680576:0@lo:38:kworker.0 00000100:00080000:3.0:1767318903.756746:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318903.756751:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318903.756754:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982680704:0@lo:38:kworker.0 00000100:00100000:3.0:1767318903.756779:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681088 00000100:00100000:3.0:1767318903.756783:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681088 00000100:00100000:3.0:1767318903.756796:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041 00000100:00100000:3.0:1767318903.756800:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982681088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318903.756822:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318903.756850:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982681088:12345-0@lo:38:kworker.0 Request processed in 49us (348us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318903.756857:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041 00000100:00080000:3.0:1767318903.756878:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318903.756882:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318903.756885:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982681088:0@lo:38:kworker.0 00000100:00100000:1.0:1767318903.761723:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982680192:12345-0@lo:400:kworker.0 Request processed in 10959us (12596us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318903.761740:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 794 00000100:00100000:0.0:1767318903.762020:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982680320:12345-0@lo:38:kworker.0 Request processed in 10959us (12616us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318903.762029:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1039 00000100:00080000:1.0:1767318903.762252:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318903.762258:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318903.762262:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982680320:0@lo:38:kworker.0 00010000:00080000:3.0:1767318903.762480:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594700 x1853163982680192/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318919 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318903.762495:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982680192:0@lo:400:kworker.0 00010000:00080000:2.0:1767318903.763258:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597100 x1853163982680448/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318919 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318903.763266:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982680448:0@lo:400:kworker.0 00010000:00010000:1.0:1767318903.941340:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318904.965351:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318905.745200:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318905.745343:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318905.754122:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915594880 00000100:00100000:1.0:1767318905.754138:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915594880 00000100:00100000:1.0:1767318905.754292:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595008 00000100:00100000:1.0:1767318905.754297:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595008 00000100:00080000:2.0:1767318905.754376:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318905.760677:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 796 00000100:00100000:0.0:1767318905.760708:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915595008:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318905.760845:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915595008:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 153us (15500us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318905.760853:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 796 00000100:00100000:0.0:1767318905.760876:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1042 00000100:00100000:0.0:1767318905.760881:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915594880:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318905.760902:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318905.760927:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915594880:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (15738us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318905.760933:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1042 00010000:00010000:2.0:1767318905.988600:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318906.308634:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767318906.308663:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982681216:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318906.308731:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318906.308880:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681216 00000100:00100000:3.0:1767318906.308891:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681216 00000100:00100000:3.0:1767318906.308945:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 550 00000100:00100000:3.0:1767318906.308952:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982681216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318906.308978:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318906.309154:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982681344:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318906.309210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318906.311719:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982681216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2642us (2863us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318906.311734:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 550 00000100:00100000:1.0:1767318906.312309:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982681216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318906.312410:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318906.312451:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681344 00000100:00100000:3.0:1767318906.312459:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681344 00000100:00100000:3.0:1767318906.312525:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 551 00000100:00100000:3.0:1767318906.312534:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324e3450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982681344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318906.312563:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318906.312624:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324e3450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982681344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (214us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318906.312631:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 551 00000100:00100000:2.0:1767318906.312876:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982681344:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318906.313256:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318907.013727:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318908.037330:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318908.869741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318908.869755:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318908.869830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318908.869835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318908.869840:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318908.869850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4152 00000100:00080000:1.0:1767318908.869857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318908.869865:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318908.869895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900e00 x1853163982681600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318908.869931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318908.869935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318908.869937:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318908.869941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4152 00000100:00080000:1.0:1767318908.869944:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318908.869948:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318908.869959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903480 x1853163982681856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318908.869968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318908.869971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318908.869973:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318908.869976:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4152 00000100:00080000:1.0:1767318908.869978:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318908.869981:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318908.869993:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900000 x1853163982681984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318908.870026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318908.870030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318908.870032:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318908.870035:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4152 00000100:00080000:1.0:1767318908.870038:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318908.870041:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318908.870054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901880 x1853163982682368/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318908.870084:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982682112:0@lo:400:kworker.0 00000100:00100000:1.0:1767318908.870458:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318908.870722:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982681472:0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.870773:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318908.870816:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681472 00000100:00100000:0.0:1767318908.870826:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681472 00000100:00100000:0.0:1767318908.870901:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 797 00000100:00100000:0.0:1767318908.870923:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982681472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.870984:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982681472:12345-0@lo:400:kworker.0 Request processed in 73us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318908.870991:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 797 00000100:00080000:0.0:1767318908.871028:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318908.871064:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982681728:0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.871092:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318908.871114:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681728 00000100:00100000:0.0:1767318908.871117:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681728 00000100:00100000:0.0:1767318908.871160:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798 00000100:00100000:0.0:1767318908.871167:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982681728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.871198:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982681728:12345-0@lo:400:kworker.0 Request processed in 31us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318908.871205:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798 00000100:00080000:0.0:1767318908.871219:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318908.873982:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901f80 x1853163982681472/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318924 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318908.874008:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982681472:0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.874041:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318908.874055:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982682240:0@lo:400:kworker.0 00000100:00100000:0.0:1767318908.874142:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767318908.874205:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901180 x1853163982681728/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318924 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318908.874216:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982681728:0@lo:400:kworker.0 00000100:00100000:3.0:1767318908.874717:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682112 00000100:00100000:3.0:1767318908.874730:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682112 00000100:00100000:3.0:1767318908.874815:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682240 00000100:00100000:3.0:1767318908.874819:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682240 00000100:00100000:3.0:1767318908.874847:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 521 00000100:00100000:3.0:1767318908.874859:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982682112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318908.874944:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982682112:12345-0@lo:400:kworker.0 Request processed in 87us (4486us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318908.874952:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 521 00000100:00100000:3.0:1767318908.874985:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 522 00000100:00100000:3.0:1767318908.874992:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982682240:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318908.875031:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982682240:12345-0@lo:400:kworker.0 Request processed in 40us (889us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318908.875038:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 522 00000100:00100000:3.0:1767318908.875095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982682240:0@lo:400:kworker.0 00000100:00100000:3.0:1767318908.875125:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982681600:0@lo:38:kworker.0 00000100:00100000:3.0:1767318908.875189:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318908.875223:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982681856:0@lo:38:kworker.0 00000100:00100000:3.0:1767318908.875249:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318908.875327:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982681984:0@lo:38:kworker.0 00000100:00100000:3.0:1767318908.875356:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318908.875395:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982682368:0@lo:38:kworker.0 00000100:00100000:3.0:1767318908.875418:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318908.875461:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681984 00000100:00100000:3.0:1767318908.875466:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681984 00000100:00100000:3.0:1767318908.875489:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 286 00000100:00100000:3.0:1767318908.875494:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982681984:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318908.875517:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318908.875555:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982681984:12345-0@lo:38:kworker.0 Request processed in 59us (198us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318908.875561:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 286 00000100:00100000:3.0:1767318908.875598:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681600 00000100:00100000:3.0:1767318908.875603:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681600 00000100:00100000:3.0:1767318908.875623:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982681856 00000100:00100000:3.0:1767318908.875626:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982681856 00000100:00100000:3.0:1767318908.875633:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682368 00000100:00100000:3.0:1767318908.875636:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682368 00000100:00100000:3.0:1767318908.875647:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043 00000100:00100000:3.0:1767318908.875652:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982681600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318908.875662:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318908.875690:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982681600:12345-0@lo:38:kworker.0 Request processed in 37us (510us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318908.875700:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043 00000100:00100000:3.0:1767318908.875711:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044 00000100:00100000:3.0:1767318908.875715:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982681856:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318908.875723:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318908.875748:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982681856:12345-0@lo:38:kworker.0 Request processed in 31us (498us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318908.875754:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044 00000100:00100000:3.0:1767318908.875767:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1045 00000100:00100000:1.0:1767318908.875772:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982682112:0@lo:400:kworker.0 00000100:00100000:3.0:1767318908.875774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982682368:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318908.875788:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318908.875812:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982682368:12345-0@lo:38:kworker.0 Request processed in 39us (393us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318908.875817:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1045 00000100:00080000:3.0:1767318908.875854:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318908.875860:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318908.875863:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982681600:0@lo:38:kworker.0 00000100:00080000:3.0:1767318908.875869:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318908.875872:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318908.875876:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982681856:0@lo:38:kworker.0 00000100:00080000:3.0:1767318908.875882:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318908.875885:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318908.875888:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982681984:0@lo:38:kworker.0 00000100:00080000:3.0:1767318908.875897:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318908.875901:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318908.875904:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982682368:0@lo:38:kworker.0 00000100:00080000:0.0:1767318908.876224:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318908.876230:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318909.060626:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318909.838120:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767318909.838224:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915595136 offset 0 length 520: 4 00000100:00100000:2.0:1767318909.838260:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318909.848214:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595392 00000100:00100000:0.0:1767318909.848239:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595392 00000100:00100000:0.0:1767318909.848346:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595264 00000100:00100000:0.0:1767318909.848349:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595264 00000100:00100000:0.0:1767318909.848374:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 799 00000100:00100000:0.0:1767318909.848385:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915595392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318909.848489:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915595392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 105us (10389us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318909.848497:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 799 00000100:00080000:1.0:1767318909.848846:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318909.849107:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1046 00000100:00100000:2.0:1767318909.849115:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915595264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318909.849142:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318909.849196:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915595264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 81us (10935us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318909.849204:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1046 00010000:00010000:1.0:1767318910.084906:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318911.110931:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318911.428667:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318911.428703:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982682496:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318911.428805:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318911.429173:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682496 00000100:00100000:1.0:1767318911.429185:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682496 00000100:00080000:0.0:1767318911.429324:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318911.429428:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318911.429441:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982682624:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318911.429496:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318911.433856:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682624 00000100:00100000:1.0:1767318911.433877:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682624 00000100:00100000:1.0:1767318911.433977:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 552 00000100:00100000:1.0:1767318911.433990:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982682496:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318911.434034:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318911.434126:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982682496:12345-0@lo:13:osp-pre-0-1.0 Request processed in 139us (5321us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318911.434134:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 552 00000100:00100000:1.0:1767318911.434154:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 553 00000100:00100000:1.0:1767318911.434160:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982682624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318911.434182:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318911.434216:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982682624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (4720us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318911.434222:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 553 00000100:00080000:1.0:1767318911.434242:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318911.434324:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982682496:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318911.437168:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982682624:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767318912.132649:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318913.156732:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318913.990552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318913.990568:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318913.990667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318913.990671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318913.990676:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318913.990687:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4157 00000100:00080000:3.0:1767318913.990693:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318913.990701:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318913.990727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5500 x1853163982682880/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318913.990764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318913.990767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318913.990769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318913.990773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4157 00000100:00080000:3.0:1767318913.990776:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318913.990780:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318913.990795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7b80 x1853163982683136/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318913.990810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767318913.990814:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318888/real 1767318888] req@ffff934631903800 x1853163982676224/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318913 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318913.990815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318913.990818:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318913.990824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4157 00000100:00080000:3.0:1767318913.990829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318913.990835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318913.990851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4e00 x1853163982683264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318913.990863:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631903800 x1853163982676224/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318913 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318913.990894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767318913.990895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982682880:0@lo:38:kworker.0 00000100:00080000:3.0:1767318913.990902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318913.990906:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318913.990910:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4157 00000100:00080000:3.0:1767318913.990913:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318913.990916:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318913.990933:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014e00 x1853163982683648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318913.990967:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982683392:0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.990994:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318913.991013:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318913.991097:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982682752:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.991135:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318913.991165:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982683008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.991190:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318913.991235:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682880 00000100:00100000:1.0:1767318913.991244:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682880 00000100:00100000:1.0:1767318913.991317:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982682752 00000100:00100000:2.0:1767318913.991322:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982683520:0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.991323:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982682752 00000100:00100000:2.0:1767318913.991363:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318913.991384:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683008 00000100:00100000:1.0:1767318913.991389:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683008 00000100:00100000:2.0:1767318913.991408:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800 00000100:00100000:2.0:1767318913.991422:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982682752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.991462:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982683136:0@lo:38:kworker.0 00000100:00100000:2.0:1767318913.991507:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982682752:12345-0@lo:400:kworker.0 Request processed in 87us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318913.991515:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318913.991545:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982683264:0@lo:38:kworker.0 00000100:00100000:0.0:1767318913.991571:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318913.991597:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318913.991613:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318913.991617:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982676224:0@lo:250:kworker.0 00000100:00100000:0.0:1767318913.991639:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982683648:0@lo:38:kworker.0 00000100:00100000:0.0:1767318913.991661:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318913.991689:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683264 00000100:00100000:0.0:1767318913.991694:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683264 00000100:00100000:0.0:1767318913.991738:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 287 00000100:00100000:0.0:1767318913.991743:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982683264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318913.991766:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318913.991801:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982683264:12345-0@lo:38:kworker.0 Request processed in 55us (228us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318913.991809:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 287 00010000:00080000:0.0:1767318913.991860:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5880 x1853163982682752/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318929 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318913.991873:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982682752:0@lo:400:kworker.0 00000100:00080000:0.0:1767318913.991897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318913.991900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318913.991903:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318913.991906:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4136 00000100:00080000:0.0:1767318913.991909:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318913.991914:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318913.991931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0e00 x1853163982683776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318913.991957:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982683776:0@lo:250:kworker.0 00000400:00000100:0.0:1767318913.991978:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982683776 offset 0 length 520: 4 00000100:00080000:0.0:1767318913.992005:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318913.992009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318913.992012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982683264:0@lo:38:kworker.0 00000100:00080000:0.0:1767318913.992046:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318913.992095:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683392 00000100:00100000:0.0:1767318913.992100:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683392 00000100:00100000:0.0:1767318913.992135:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683520 00000100:00100000:0.0:1767318913.992139:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683520 00000100:00100000:0.0:1767318913.992161:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 523 00000100:00100000:0.0:1767318913.992168:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982683392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.992203:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982683392:12345-0@lo:400:kworker.0 Request processed in 36us (1190us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318913.992210:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 523 00000100:00100000:0.0:1767318913.992221:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 524 00000100:00100000:0.0:1767318913.992227:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982683520:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318913.992252:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982683520:12345-0@lo:400:kworker.0 Request processed in 26us (890us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318913.992258:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 524 00000100:00100000:0.0:1767318913.992303:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683136 00000100:00100000:0.0:1767318913.992307:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683136 00000100:00100000:1.0:1767318913.992545:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683648 00000100:00100000:1.0:1767318913.992550:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683648 00000100:00100000:1.0:1767318913.992565:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 801 00000100:00100000:1.0:1767318913.992573:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982683008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318913.992616:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982683008:12345-0@lo:400:kworker.0 Request processed in 45us (1425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318913.992622:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 801 00000100:00100000:1.0:1767318913.992638:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047 00000100:00100000:1.0:1767318913.992644:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982682880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318913.992657:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318913.992689:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982682880:12345-0@lo:38:kworker.0 Request processed in 43us (1697us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318913.992696:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047 00000100:00100000:1.0:1767318913.992789:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048 00000100:00100000:1.0:1767318913.992794:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982683648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318913.992804:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318913.992831:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982683648:12345-0@lo:38:kworker.0 Request processed in 36us (1169us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318913.992837:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048 00010000:00080000:1.0:1767318913.992870:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4a80 x1853163982683008/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318929 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318913.992884:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982683008:0@lo:400:kworker.0 00000100:00080000:1.0:1767318913.992911:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318913.992915:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318913.992919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982683648:0@lo:38:kworker.0 00000100:00080000:1.0:1767318913.992927:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318913.992931:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318913.992933:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982682880:0@lo:38:kworker.0 00000100:00080000:1.0:1767318913.992966:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318913.992969:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318913.993037:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982683520:0@lo:400:kworker.0 00000100:00100000:3.0:1767318913.993047:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982683392:0@lo:400:kworker.0 00000100:00100000:2.0:1767318913.994000:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800 00000100:00100000:0.0:1767318913.997949:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049 00000100:00100000:0.0:1767318913.997966:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982683136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318913.997995:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318913.998125:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982683136:12345-0@lo:38:kworker.0 Request processed in 160us (6611us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318913.998134:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049 00000100:00080000:1.0:1767318913.998447:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318913.998455:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318913.998460:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982683136:0@lo:38:kworker.0 00010000:00010000:1.0:1767318914.185658:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318914.959593:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318914.959892:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318914.961008:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595520 00000100:00100000:0.0:1767318914.961029:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595520 00000100:00100000:0.0:1767318914.961182:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595648 00000100:00100000:0.0:1767318914.961187:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595648 00000100:00100000:0.0:1767318914.961254:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 802 00000100:00100000:0.0:1767318914.961266:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915595648:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318914.961509:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1050 00000100:00100000:1.0:1767318914.961517:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915595520:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318914.961544:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318914.961590:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915595520:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 72us (2014us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318914.961671:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1050 00000100:00100000:0.0:1767318914.962165:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915595648:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 896us (2268us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318914.962174:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 802 00000100:00080000:2.0:1767318914.962263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767318915.206075:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318916.228625:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318916.550003:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982683904:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318916.550123:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318916.551642:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982683904 00000100:00100000:2.0:1767318916.551662:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982683904 00000100:00100000:2.0:1767318916.551781:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 554 00000100:00100000:2.0:1767318916.551793:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982683904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318916.551840:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318916.551933:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982683904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 142us (1809us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318916.551941:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 554 00000100:00100000:0.0:1767318916.552316:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982683904:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767318916.552412:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318916.554314:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982684032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318916.554404:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318916.554715:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684032 00000100:00100000:0.0:1767318916.554727:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684032 00000100:00100000:0.0:1767318916.554815:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 555 00000100:00100000:0.0:1767318916.554826:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982684032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318916.554862:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318916.554921:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982684032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (516us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318916.554929:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 555 00000100:00080000:3.0:1767318916.555003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318916.555091:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982684032:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318917.252862:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318918.276614:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318919.108660:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318919.108672:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318919.108741:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318919.108745:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318919.108749:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318919.108759:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4162 00000100:00080000:2.0:1767318919.108764:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318919.108771:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318919.108794:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982684288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318919.108840:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318919.108843:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318919.108845:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318919.108849:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4162 00000100:00080000:2.0:1767318919.108851:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318919.108854:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318919.108865:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962680 x1853163982684544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318919.108875:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318919.108878:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318919.108880:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318919.108884:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4162 00000100:00080000:2.0:1767318919.108886:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318919.108890:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318919.108901:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982684672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318919.108938:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318919.108941:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318919.108943:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318919.108947:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4162 00000100:00080000:2.0:1767318919.108949:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318919.108952:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318919.108961:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163982685056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318919.108985:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982684416:0@lo:400:kworker.0 00000100:00100000:2.0:1767318919.109054:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318919.109100:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318919.109105:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982684800:0@lo:400:kworker.0 00000100:00100000:2.0:1767318919.109129:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318919.109300:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982684160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109327:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318919.109359:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767318919.109365:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982684928:0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109401:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318919.109434:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684416 00000100:00100000:3.0:1767318919.109451:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684416 00000100:00100000:3.0:1767318919.109518:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684160 00000100:00100000:3.0:1767318919.109522:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684160 00000100:00100000:3.0:1767318919.109548:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 803 00000100:00100000:3.0:1767318919.109557:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982684416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109611:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982684416:12345-0@lo:400:kworker.0 Request processed in 55us (556us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318919.109618:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 803 00000100:00100000:3.0:1767318919.109636:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804 00000100:00100000:3.0:1767318919.109643:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982684160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109673:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982684160:12345-0@lo:400:kworker.0 Request processed in 31us (345us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318919.109679:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804 00000100:00100000:3.0:1767318919.109700:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684800 00000100:00100000:3.0:1767318919.109703:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684800 00000100:00100000:3.0:1767318919.109727:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684928 00000100:00100000:3.0:1767318919.109730:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684928 00000100:00100000:3.0:1767318919.109747:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 525 00000100:00100000:3.0:1767318919.109752:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982684800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109781:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982684800:12345-0@lo:400:kworker.0 Request processed in 28us (651us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318919.109786:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 525 00000100:00100000:3.0:1767318919.109798:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 526 00000100:00100000:3.0:1767318919.109803:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982684928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109829:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982684928:12345-0@lo:400:kworker.0 Request processed in 26us (427us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318919.109834:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 526 00000100:00080000:3.0:1767318919.109849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318919.109853:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318919.109855:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318919.109892:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982684928:0@lo:400:kworker.0 00010000:00080000:3.0:1767318919.109904:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163982684160/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318935 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318919.109927:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982684160:0@lo:400:kworker.0 00000100:00100000:3.0:1767318919.109955:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982684800:0@lo:400:kworker.0 00010000:00080000:3.0:1767318919.109963:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962300 x1853163982684416/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318935 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318919.109971:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982684416:0@lo:400:kworker.0 00000100:00100000:0.0:1767318919.110482:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982684288:0@lo:38:kworker.0 00000100:00100000:0.0:1767318919.110521:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318919.110552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982684544:0@lo:38:kworker.0 00000100:00100000:0.0:1767318919.110572:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318919.110594:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982684672:0@lo:38:kworker.0 00000100:00100000:0.0:1767318919.110618:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318919.110621:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684288 00000100:00100000:2.0:1767318919.110627:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684288 00000100:00100000:0.0:1767318919.110644:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982685056:0@lo:38:kworker.0 00000100:00100000:2.0:1767318919.110652:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684544 00000100:00100000:2.0:1767318919.110654:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684544 00000100:00100000:0.0:1767318919.110664:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318919.110728:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982684672 00000100:00100000:1.0:1767318919.110733:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982684672 00000100:00100000:1.0:1767318919.110757:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 288 00000100:00100000:1.0:1767318919.110764:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982684672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318919.110783:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318919.110826:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982684672:12345-0@lo:38:kworker.0 Request processed in 61us (206us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318919.110832:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 288 00000100:00080000:1.0:1767318919.110884:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318919.110889:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318919.110892:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982684672:0@lo:38:kworker.0 00000100:00100000:2.0:1767318919.114132:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685056 00000100:00100000:2.0:1767318919.114148:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685056 00000100:00100000:2.0:1767318919.114204:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051 00000100:00100000:2.0:1767318919.114212:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982684288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318919.114531:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318919.114632:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982684288:12345-0@lo:38:kworker.0 Request processed in 419us (4109us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318919.114641:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051 00000100:00100000:2.0:1767318919.114662:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052 00000100:00100000:2.0:1767318919.114666:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982684544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318919.114676:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318919.114701:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982684544:12345-0@lo:38:kworker.0 Request processed in 33us (4128us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318919.114707:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052 00000100:00100000:2.0:1767318919.114718:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053 00000100:00100000:2.0:1767318919.114722:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982685056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318919.114731:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318919.114754:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982685056:12345-0@lo:38:kworker.0 Request processed in 30us (4089us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318919.114759:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053 00000100:00080000:2.0:1767318919.114829:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318919.114835:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318919.114839:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982684288:0@lo:38:kworker.0 00000100:00080000:2.0:1767318919.114851:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318919.114854:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318919.114857:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982684544:0@lo:38:kworker.0 00000100:00080000:2.0:1767318919.114865:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318919.114869:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318919.114873:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982685056:0@lo:38:kworker.0 00010000:00010000:1.0:1767318919.300688:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318920.083420:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318920.083518:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318920.085619:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595776 00000100:00100000:3.0:1767318920.085637:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595776 00000100:00100000:2.0:1767318920.085709:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915595904 00000100:00100000:2.0:1767318920.085714:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915595904 00000100:00080000:0.0:1767318920.085832:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318920.086145:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 805 00000100:00100000:2.0:1767318920.086157:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915595904:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318920.086248:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915595904:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (2729us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318920.086256:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 805 00000100:00100000:2.0:1767318920.086303:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1054 00000100:00100000:2.0:1767318920.086307:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915595776:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318920.086326:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318920.086366:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915595776:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (2945us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318920.086373:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1054 00010000:00010000:1.0:1767318920.328372:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318921.348927:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318921.668797:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318921.668814:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982685312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318921.668882:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318921.669029:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685312 00000100:00100000:3.0:1767318921.669038:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685312 00000100:00100000:3.0:1767318921.669104:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 556 00000100:00100000:3.0:1767318921.669114:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982685312:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318921.669150:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00002000:00000024:3.0:1767318921.669153:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318921.669163:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982685184:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318921.669198:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318921.669227:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982685312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 111us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318921.669231:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685184 00000100:00100000:1.0:1767318921.669236:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685184 00000100:00100000:3.0:1767318921.669237:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 556 00000100:00100000:3.0:1767318921.669265:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 557 00000100:00080000:2.0:1767318921.669297:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767318921.669302:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318921.669309:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982685184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318921.669337:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318921.669350:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982685312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318921.669376:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982685184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318921.671440:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982685184:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318921.680018:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 557 00010000:00010000:1.0:1767318922.373231:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318923.396521:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318924.228430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318924.228441:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318924.228528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318924.228535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318924.228539:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318924.228547:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4167 00000100:00080000:2.0:1767318924.228553:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318924.228560:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318924.228585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c000 x1853163982685568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318924.228621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318924.228625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318924.228627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318924.228630:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4167 00000100:00080000:2.0:1767318924.228633:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318924.228637:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318924.228648:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163982685824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318924.228656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318924.228659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318924.228661:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318924.228664:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4167 00000100:00080000:2.0:1767318924.228666:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318924.228670:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318924.228681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163982685952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318924.228715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318924.228720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318924.228722:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318924.228725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4167 00000100:00080000:2.0:1767318924.228727:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318924.228731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318924.228743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246300 x1853163982686336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318924.228774:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982685696:0@lo:400:kworker.0 00000100:00100000:2.0:1767318924.228854:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318924.228899:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767318924.228905:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982686080:0@lo:400:kworker.0 00000100:00100000:2.0:1767318924.228933:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318924.228965:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685696 00000100:00100000:2.0:1767318924.228972:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685696 00000100:00100000:2.0:1767318924.229033:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806 00000100:00100000:2.0:1767318924.229045:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982685696:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318924.229094:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982685696:12345-0@lo:400:kworker.0 Request processed in 51us (238us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318924.229101:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806 00000100:00080000:2.0:1767318924.229119:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318924.229148:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36680 x1853163982685696/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318940 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318924.229161:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982685696:0@lo:400:kworker.0 00000100:00100000:3.0:1767318924.229312:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982685440:0@lo:400:kworker.0 00000100:00100000:3.0:1767318924.229343:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318924.229373:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767318924.229378:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982686208:0@lo:400:kworker.0 00000100:00100000:3.0:1767318924.229401:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318924.229490:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685440 00000100:00100000:1.0:1767318924.229495:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685440 00000100:00100000:1.0:1767318924.229541:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807 00000100:00100000:1.0:1767318924.229548:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982685440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318924.229590:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982685440:12345-0@lo:400:kworker.0 Request processed in 43us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318924.229597:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807 00010000:00080000:1.0:1767318924.229624:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ce00 x1853163982685440/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318940 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318924.229632:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982685440:0@lo:400:kworker.0 00000100:00100000:1.0:1767318924.229658:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982685568:0@lo:38:kworker.0 00000100:00100000:1.0:1767318924.229682:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318924.229698:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982685824:0@lo:38:kworker.0 00000100:00100000:1.0:1767318924.229717:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318924.229731:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982685952:0@lo:38:kworker.0 00000100:00100000:1.0:1767318924.229753:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318924.229785:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982686336:0@lo:38:kworker.0 00000100:00100000:1.0:1767318924.229806:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318924.229836:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685952 00000100:00100000:1.0:1767318924.229839:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685952 00000100:00100000:1.0:1767318924.229859:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 289 00000100:00100000:1.0:1767318924.229864:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982685952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318924.229881:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318924.229912:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982685952:12345-0@lo:38:kworker.0 Request processed in 46us (158us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318924.229918:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 289 00000100:00100000:1.0:1767318924.229935:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685568 00000100:00100000:1.0:1767318924.229938:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685568 00000100:00100000:1.0:1767318924.229948:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982685824 00000100:00100000:1.0:1767318924.229950:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982685824 00000100:00100000:1.0:1767318924.229957:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686336 00000100:00100000:1.0:1767318924.229959:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686336 00000100:00100000:1.0:1767318924.229969:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055 00000100:00100000:1.0:1767318924.229973:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982685568:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318924.229982:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318924.230004:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982685568:12345-0@lo:38:kworker.0 Request processed in 30us (321us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318924.230010:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055 00000100:00100000:1.0:1767318924.230019:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056 00000100:00100000:1.0:1767318924.230023:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982685824:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318924.230032:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318924.230053:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982685824:12345-0@lo:38:kworker.0 Request processed in 28us (336us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318924.230058:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056 00000100:00100000:1.0:1767318924.230068:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057 00000100:00100000:1.0:1767318924.230072:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982686336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318924.230080:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318924.230099:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982686336:12345-0@lo:38:kworker.0 Request processed in 26us (292us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318924.230104:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057 00000100:00080000:1.0:1767318924.230130:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318924.230135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318924.230138:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982685568:0@lo:38:kworker.0 00000100:00080000:1.0:1767318924.230146:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318924.230149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318924.230152:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982685824:0@lo:38:kworker.0 00000100:00080000:1.0:1767318924.230158:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318924.230161:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318924.230163:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982685952:0@lo:38:kworker.0 00000100:00080000:1.0:1767318924.230170:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318924.230172:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318924.230175:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982686336:0@lo:38:kworker.0 00000100:00080000:3.0:1767318924.230253:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318924.230734:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686080 00000100:00100000:0.0:1767318924.230740:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686080 00000100:00100000:0.0:1767318924.230800:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686208 00000100:00100000:0.0:1767318924.230804:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686208 00000100:00100000:0.0:1767318924.230838:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 527 00000100:00100000:0.0:1767318924.230847:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982686080:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318924.230906:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982686080:12345-0@lo:400:kworker.0 Request processed in 61us (1972us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318924.230913:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 527 00000100:00100000:0.0:1767318924.230928:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 528 00000100:00100000:0.0:1767318924.230933:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982686208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318924.230964:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982686208:12345-0@lo:400:kworker.0 Request processed in 31us (1562us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318924.230969:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 528 00000100:00100000:0.0:1767318924.231005:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982686080:0@lo:400:kworker.0 00000100:00100000:0.0:1767318924.231032:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982686208:0@lo:400:kworker.0 00000100:00080000:3.0:1767318924.231177:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318924.231181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318924.420502:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318925.203024:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318925.203160:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318925.216955:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596160 00000100:00100000:0.0:1767318925.216974:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596160 00000100:00100000:0.0:1767318925.220746:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 808 00000100:00100000:0.0:1767318925.220777:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915596160:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318925.220876:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915596160:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 117us (17865us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318925.220884:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 808 00000100:00100000:0.0:1767318925.220912:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596032 00000100:00100000:0.0:1767318925.220919:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596032 00000100:00080000:2.0:1767318925.221476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767318925.223989:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1058 00000100:00100000:3.0:1767318925.224001:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915596032:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318925.224027:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318925.224099:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915596032:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 98us (20938us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318925.224107:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1058 00010000:00010000:1.0:1767318925.445848:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318926.469117:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318926.788532:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1767318926.788557:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982686464:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318926.788646:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318926.788718:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982686592:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318926.788741:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318926.788784:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686464 00000100:00100000:2.0:1767318926.788792:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686464 00000100:00100000:1.0:1767318926.788882:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686592 00000100:00100000:1.0:1767318926.788888:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686592 00000100:00100000:1.0:1767318926.788944:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 558 00000100:00100000:1.0:1767318926.788954:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982686592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318926.788984:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767318926.789037:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767318926.789042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318926.789046:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982686592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (303us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318926.789053:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 558 00000100:00100000:3.0:1767318926.789080:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982686592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318926.789177:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 559 00000100:00100000:2.0:1767318926.789186:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982686464:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767318926.789203:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318926.789254:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982686464:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (607us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318926.789259:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 559 00000100:00100000:3.0:1767318926.789345:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982686464:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767318927.492525:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318928.516876:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318929.348817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318929.348828:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318929.348910:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767318929.348929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318929.348933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318929.348937:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318929.348945:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4172 00000100:00080000:3.0:1767318929.348949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318929.348956:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318929.348978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962300 x1853163982686848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318929.349046:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982686848:0@lo:38:kworker.0 00000100:00100000:2.0:1767318929.349111:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318929.349199:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686848 00000100:00100000:2.0:1767318929.349207:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686848 00000100:00100000:2.0:1767318929.349244:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059 00000100:00100000:2.0:1767318929.349252:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982686848:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318929.349286:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318929.349326:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982686848:12345-0@lo:38:kworker.0 Request processed in 74us (215us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318929.349334:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059 00000100:00080000:2.0:1767318929.349370:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318929.349375:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318929.349379:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982686848:0@lo:38:kworker.0 00000100:00100000:0.0:1767318929.352245:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318929.352301:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982686976:0@lo:400:kworker.0 00000100:00100000:0.0:1767318929.352372:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318929.352464:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686976 00000100:00100000:0.0:1767318929.352472:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686976 00000100:00100000:0.0:1767318929.352532:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 809 00000100:00100000:0.0:1767318929.352542:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982686976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318929.352604:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982686976:12345-0@lo:400:kworker.0 Request processed in 63us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318929.352612:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 809 00010000:00080000:0.0:1767318929.352653:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163982686976/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318945 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318929.352667:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982686976:0@lo:400:kworker.0 00000100:00080000:2.0:1767318929.353257:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318929.353448:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982686720:0@lo:400:kworker.0 00000100:00100000:1.0:1767318929.353489:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318929.353562:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982686720 00000100:00100000:2.0:1767318929.353568:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982686720 00000100:00100000:2.0:1767318929.353629:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 810 00000100:00100000:2.0:1767318929.353636:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982686720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318929.353675:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982686720:12345-0@lo:400:kworker.0 Request processed in 41us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318929.353681:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 810 00000100:00080000:0.0:1767318929.353775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318929.353861:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963b80 x1853163982686720/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318945 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318929.353870:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982686720:0@lo:400:kworker.0 00000100:00080000:3.0:1767318929.356026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318929.356033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318929.356036:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318929.356042:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4172 00000100:00080000:3.0:1767318929.356046:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318929.356051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318929.356084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982687104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318929.356102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318929.356105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318929.356106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318929.356109:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4172 00000100:00080000:3.0:1767318929.356111:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318929.356113:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318929.356120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960000 x1853163982687232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318929.356153:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982687360:0@lo:400:kworker.0 00000100:00100000:3.0:1767318929.356241:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318929.356338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318929.356341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318929.356342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318929.356345:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4172 00000100:00080000:3.0:1767318929.356347:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318929.356349:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318929.356356:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982687488:0@lo:400:kworker.0 00000100:00100000:3.0:1767318929.356374:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318929.356399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0a80 x1853163982687616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318929.356467:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687360 00000100:00100000:1.0:1767318929.356473:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687360 00000100:00100000:1.0:1767318929.356515:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687488 00000100:00100000:1.0:1767318929.356517:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687488 00000100:00100000:1.0:1767318929.356540:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 529 00000100:00100000:1.0:1767318929.356546:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982687360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318929.356600:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982687360:12345-0@lo:400:kworker.0 Request processed in 54us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318929.356607:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 529 00000100:00100000:1.0:1767318929.356623:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 530 00000100:00100000:1.0:1767318929.356630:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982687488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318929.356660:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982687488:12345-0@lo:400:kworker.0 Request processed in 31us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318929.356664:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 530 00000100:00100000:1.0:1767318929.356692:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982687488:0@lo:400:kworker.0 00000100:00100000:1.0:1767318929.356711:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982687360:0@lo:400:kworker.0 00000100:00100000:2.0:1767318929.356897:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982687104:0@lo:38:kworker.0 00000100:00100000:2.0:1767318929.356922:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318929.356946:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982687232:0@lo:38:kworker.0 00000100:00100000:2.0:1767318929.356965:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318929.356985:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982687616:0@lo:38:kworker.0 00000100:00100000:2.0:1767318929.356997:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318929.357017:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687104 00000100:00100000:2.0:1767318929.357019:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687104 00000100:00100000:2.0:1767318929.357030:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687616 00000100:00100000:2.0:1767318929.357032:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687616 00000100:00100000:2.0:1767318929.357041:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060 00000100:00100000:2.0:1767318929.357044:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982687104:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318929.357058:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318929.357079:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982687104:12345-0@lo:38:kworker.0 Request processed in 34us (156us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318929.357083:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060 00000100:00100000:2.0:1767318929.357090:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061 00000100:00100000:2.0:1767318929.357092:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982687616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318929.357098:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318929.357110:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982687616:12345-0@lo:38:kworker.0 Request processed in 16us (112us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318929.357114:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061 00000100:00080000:2.0:1767318929.357130:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318929.357133:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318929.357135:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982687104:0@lo:38:kworker.0 00000100:00080000:2.0:1767318929.357140:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318929.357142:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318929.357144:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982687616:0@lo:38:kworker.0 00000100:00100000:0.0:1767318929.357215:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687232 00000100:00100000:0.0:1767318929.357220:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687232 00000100:00100000:0.0:1767318929.357255:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 290 00000100:00100000:0.0:1767318929.357260:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d87050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982687232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318929.357288:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318929.357331:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d87050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982687232:12345-0@lo:38:kworker.0 Request processed in 69us (363us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318929.357335:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 290 00000100:00080000:1.0:1767318929.357403:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318929.357408:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318929.357411:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982687232:0@lo:38:kworker.0 00000100:00080000:3.0:1767318929.357675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767318929.357679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318929.541675:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318930.319419:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318930.319519:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318930.319629:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596416 00000100:00100000:2.0:1767318930.319640:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596416 00000100:00100000:2.0:1767318930.319714:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596288 00000100:00100000:2.0:1767318930.319718:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596288 00000100:00100000:2.0:1767318930.319741:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 811 00000100:00080000:3.0:1767318930.319779:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318930.319828:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915596416:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318930.319903:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915596416:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 153us (492us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318930.322702:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1062 00000100:00100000:1.0:1767318930.322716:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915596288:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318930.322742:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318930.322813:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915596288:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 98us (3293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318930.322821:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1062 00000100:00100000:2.0:1767318930.330615:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 811 00010000:00010000:2.0:1767318930.566462:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318931.590108:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318931.908869:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982687744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318931.908940:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318931.909074:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687744 00000100:00100000:1.0:1767318931.909082:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687744 00000100:00100000:1.0:1767318931.909162:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 560 00000100:00100000:1.0:1767318931.909172:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982687744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318931.909200:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318931.909249:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982687744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318931.909256:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 560 00000100:00080000:0.0:1767318931.909497:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318931.909557:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982687744:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318931.909815:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767318931.909825:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982687872:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318931.909854:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318931.909977:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982687872 00000100:00100000:1.0:1767318931.909983:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982687872 00000100:00100000:1.0:1767318931.910027:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 561 00000100:00100000:1.0:1767318931.910034:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982687872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318931.910053:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318931.910096:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982687872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 63us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318931.910103:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 561 00000100:00080000:0.0:1767318931.910162:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318931.910220:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982687872:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318932.612969:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318933.639503:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318934.469992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318934.470000:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767318934.470091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982688000:0@lo:400:kworker.0 00000100:00100000:1.0:1767318934.470159:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318934.470246:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688000 00000100:00100000:0.0:1767318934.470254:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688000 00000100:00100000:0.0:1767318934.470336:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 812 00000100:00100000:0.0:1767318934.470346:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982688000:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767318934.470438:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318934.470558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318934.470564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318934.470567:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318934.470572:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4177 00000100:00080000:3.0:1767318934.470576:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318934.470581:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318934.470604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0a80 x1853163982688128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318934.470629:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982688128:0@lo:38:kworker.0 00000100:00100000:3.0:1767318934.470658:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318934.470689:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688128 00000100:00100000:3.0:1767318934.470693:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688128 00000100:00100000:3.0:1767318934.470715:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1063 00000100:00100000:3.0:1767318934.470720:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688128:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318934.470738:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318934.470770:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688128:12345-0@lo:38:kworker.0 Request processed in 48us (111us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318934.470776:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1063 00000100:00080000:3.0:1767318934.470811:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318934.470816:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318934.470819:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982688128:0@lo:38:kworker.0 00000100:00100000:2.0:1767318934.471015:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982688256:0@lo:400:kworker.0 00000100:00100000:2.0:1767318934.471044:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318934.471109:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688256 00000100:00100000:2.0:1767318934.471113:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688256 00000100:00100000:2.0:1767318934.471141:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 813 00000100:00100000:2.0:1767318934.471148:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982688256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318934.471185:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982688256:12345-0@lo:400:kworker.0 Request processed in 38us (141us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318934.471191:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 813 00000100:00080000:2.0:1767318934.471205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767318934.471221:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3b80 x1853163982688256/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318950 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318934.471232:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982688256:0@lo:400:kworker.0 00000100:00100000:0.0:1767318934.471669:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982688000:12345-0@lo:400:kworker.0 Request processed in 1324us (1509us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318934.471676:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 812 00000100:00080000:3.0:1767318934.471728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318934.471733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318934.471735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318934.471739:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4177 00000100:00080000:3.0:1767318934.471742:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318934.471746:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318934.471764:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0380 x1853163982688384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318934.471786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318934.471789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318934.471791:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318934.471794:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4177 00000100:00080000:3.0:1767318934.471796:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318934.471800:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318934.471811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982688512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318934.471852:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982688640:0@lo:400:kworker.0 00000100:00100000:3.0:1767318934.471883:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318934.471915:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688640 00000100:00100000:3.0:1767318934.471919:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688640 00000100:00100000:3.0:1767318934.471952:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 531 00000100:00100000:3.0:1767318934.471958:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982688640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318934.471991:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982688640:12345-0@lo:400:kworker.0 Request processed in 33us (107us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318934.471997:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 531 00000100:00080000:3.0:1767318934.472016:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318934.472033:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982688640:0@lo:400:kworker.0 00000100:00080000:3.0:1767318934.472070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318934.472074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318934.472076:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318934.472080:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4177 00000100:00080000:3.0:1767318934.472091:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318934.472096:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318934.472104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982688768:0@lo:400:kworker.0 00000100:00100000:3.0:1767318934.472127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318934.472151:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688768 00000100:00100000:3.0:1767318934.472154:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688768 00000100:00100000:3.0:1767318934.472178:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 532 00000100:00100000:3.0:1767318934.472184:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982688768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318934.472217:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982688768:12345-0@lo:400:kworker.0 Request processed in 33us (89us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318934.472223:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 532 00000100:00080000:3.0:1767318934.472239:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318934.472254:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982688768:0@lo:400:kworker.0 00000100:00100000:3.0:1767318934.472296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2300 x1853163982688896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318934.472521:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982688384:0@lo:38:kworker.0 00000100:00100000:0.0:1767318934.472550:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318934.472573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982688512:0@lo:38:kworker.0 00000100:00100000:0.0:1767318934.472597:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318934.472619:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982688896:0@lo:38:kworker.0 00000100:00100000:0.0:1767318934.472637:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318934.472661:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688512 00000100:00100000:0.0:1767318934.472665:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688512 00000100:00100000:0.0:1767318934.472682:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 291 00000100:00100000:0.0:1767318934.472687:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982688512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318934.472702:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318934.472732:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982688512:12345-0@lo:38:kworker.0 Request processed in 43us (133us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318934.472738:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 291 00000100:00080000:0.0:1767318934.472767:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318934.472770:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318934.472773:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982688512:0@lo:38:kworker.0 00000100:00100000:0.0:1767318934.472789:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688384 00000100:00100000:0.0:1767318934.472793:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688384 00000100:00100000:0.0:1767318934.472810:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982688896 00000100:00100000:0.0:1767318934.472812:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982688896 00000100:00100000:0.0:1767318934.472826:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064 00000100:00100000:0.0:1767318934.472830:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318934.472838:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318934.472865:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688384:12345-0@lo:38:kworker.0 Request processed in 34us (315us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318934.472870:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064 00000100:00100000:0.0:1767318934.472881:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065 00000100:00100000:0.0:1767318934.472884:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318934.472891:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318934.472910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982688896:12345-0@lo:38:kworker.0 Request processed in 25us (272us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318934.472914:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065 00000100:00080000:0.0:1767318934.472935:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318934.472939:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318934.472941:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982688384:0@lo:38:kworker.0 00000100:00080000:0.0:1767318934.472948:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318934.472951:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318934.472953:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982688896:0@lo:38:kworker.0 00010000:00080000:1.0:1767318934.473605:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c2a00 x1853163982688000/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318950 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318934.473618:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982688000:0@lo:400:kworker.0 00010000:00010000:1.0:1767318934.660434:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318935.437389:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318935.437624:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915596544 offset 0 length 520: 4 00000100:00100000:1.0:1767318935.437650:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318935.445866:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596800 00000100:00100000:3.0:1767318935.445882:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596800 00000100:00100000:3.0:1767318935.446421:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596672 00000100:00100000:3.0:1767318935.446426:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596672 00000100:00100000:3.0:1767318935.446460:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 814 00000100:00100000:3.0:1767318935.446471:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915596800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318935.446560:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915596800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (9182us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318935.446568:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 814 00000100:00100000:3.0:1767318935.446588:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1066 00000100:00100000:3.0:1767318935.446593:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915596672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767318935.446614:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318935.446640:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915596672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (8990us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318935.446646:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1066 00000100:00080000:1.0:1767318935.446886:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318935.689892:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318936.711648:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318937.028840:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318937.028863:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982689024:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318937.028932:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318937.029078:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689024 00000100:00100000:0.0:1767318937.029087:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689024 00000100:00100000:0.0:1767318937.029155:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 562 00000100:00100000:0.0:1767318937.029166:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982689024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318937.029197:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318937.029247:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982689024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (315us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318937.029254:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 562 00000100:00100000:2.0:1767318937.029452:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982689024:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318937.029767:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318937.032297:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982689152:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318937.032397:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318937.032516:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689152 00000100:00100000:1.0:1767318937.032527:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689152 00000100:00100000:1.0:1767318937.032606:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 563 00000100:00100000:1.0:1767318937.032616:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982689152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318937.032663:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767318937.034483:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318937.043575:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982689152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10947us (11164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318937.043597:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 563 00000100:00100000:0.0:1767318937.047231:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982689152:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318937.733446:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318938.564559:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318913/real 1767318913] req@ffff93463e6f0e00 x1853163982683776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318938 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318938.564595:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463e6f0e00 x1853163982683776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318938 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318938.564614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318938.564651:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318938.564654:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982683776:0@lo:250:kworker.0 00000100:00080000:1.0:1767318938.565327:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318938.565334:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767318938.565338:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318938.565343:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4162 00000100:00080000:1.0:1767318938.565347:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318938.565353:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318938.565386:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163982689280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318938.565450:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318938.565454:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318938.565456:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318938.565460:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4183 00000100:00080000:1.0:1767318938.565462:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318938.565466:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318938.565484:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962300 x1853163982689536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318938.565500:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318938.565503:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318938.565505:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318938.565508:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4183 00000100:00080000:1.0:1767318938.565511:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318938.565514:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318938.565525:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163982689792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318938.565534:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318938.565537:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318938.565538:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318938.565542:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4183 00000100:00080000:1.0:1767318938.565544:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318938.565548:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318938.565558:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90a80 x1853163982689920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318938.565590:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318938.565594:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318938.565596:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318938.565599:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4183 00000100:00080000:1.0:1767318938.565601:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318938.565605:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318938.565616:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246d80 x1853163982690304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318938.565659:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982690176:0@lo:400:kworker.0 00000100:00100000:1.0:1767318938.565717:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318938.565777:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318938.565783:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982690048:0@lo:400:kworker.0 00000100:00100000:1.0:1767318938.565806:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318938.566188:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982689664:0@lo:400:kworker.0 00000100:00100000:2.0:1767318938.566223:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318938.566254:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318938.566260:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982689408:0@lo:400:kworker.0 00000100:00100000:2.0:1767318938.566314:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318938.566538:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690176 00000100:00100000:3.0:1767318938.566546:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690176 00000100:00100000:3.0:1767318938.566598:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690048 00000100:00100000:3.0:1767318938.566602:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690048 00000100:00100000:3.0:1767318938.566634:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 533 00000100:00100000:3.0:1767318938.566643:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982690176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.566700:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982690176:12345-0@lo:400:kworker.0 Request processed in 58us (983us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318938.566707:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 533 00000100:00100000:3.0:1767318938.566724:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 534 00000100:00100000:3.0:1767318938.566731:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982690048:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.566758:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982690048:12345-0@lo:400:kworker.0 Request processed in 28us (951us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318938.566763:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 534 00000100:00100000:3.0:1767318938.566815:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982690048:0@lo:400:kworker.0 00000100:00100000:3.0:1767318938.566825:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982690176:0@lo:400:kworker.0 00000100:00100000:0.0:1767318938.567088:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689664 00000100:00100000:0.0:1767318938.567093:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689664 00000100:00100000:0.0:1767318938.567144:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689408 00000100:00100000:0.0:1767318938.567148:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689408 00000100:00100000:0.0:1767318938.567171:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 815 00000100:00100000:0.0:1767318938.567177:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982689664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318938.567219:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982689664:12345-0@lo:400:kworker.0 Request processed in 42us (995us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318938.567225:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 815 00000100:00100000:0.0:1767318938.567238:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 816 00000100:00100000:0.0:1767318938.567244:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982689408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318938.567299:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982689408:12345-0@lo:400:kworker.0 Request processed in 54us (984us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318938.567304:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 816 00010000:00080000:0.0:1767318938.567336:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963b80 x1853163982689408/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318954 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318938.567347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982689408:0@lo:400:kworker.0 00010000:00080000:0.0:1767318938.567358:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163982689664/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318954 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318938.567365:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982689664:0@lo:400:kworker.0 00000100:00100000:0.0:1767318938.567385:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982689280:0@lo:250:kworker.0 00000400:00000100:0.0:1767318938.567408:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982689280 offset 0 length 520: 4 00000100:00100000:0.0:1767318938.567429:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982689536:0@lo:38:kworker.0 00000100:00100000:0.0:1767318938.568023:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318938.568156:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689536 00000100:00100000:1.0:1767318938.568162:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689536 00000100:00100000:1.0:1767318938.568192:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067 00000100:00100000:1.0:1767318938.568198:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982689536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318938.568217:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318938.568253:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982689536:12345-0@lo:38:kworker.0 Request processed in 54us (231us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318938.568260:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067 00000100:00080000:0.0:1767318938.568404:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318938.568410:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318938.568413:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982689536:0@lo:38:kworker.0 00000100:00100000:0.0:1767318938.568420:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982689792:0@lo:38:kworker.0 00000100:00100000:0.0:1767318938.568518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767318938.568594:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318938.568645:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982689920:0@lo:38:kworker.0 00000100:00100000:0.0:1767318938.568672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318938.568697:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982690304:0@lo:38:kworker.0 00000100:00100000:0.0:1767318938.568719:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318938.568760:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689920 00000100:00100000:1.0:1767318938.568765:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689920 00000100:00100000:1.0:1767318938.568795:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 292 00000100:00100000:1.0:1767318938.568801:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982689920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318938.568811:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318938.568838:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982689920:12345-0@lo:38:kworker.0 Request processed in 37us (166us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318938.568845:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 292 00000100:00080000:0.0:1767318938.578054:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318938.578067:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318938.578071:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982689920:0@lo:38:kworker.0 00000100:00100000:2.0:1767318938.578313:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982689792 00000100:00100000:2.0:1767318938.578321:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982689792 00000100:00100000:1.0:1767318938.578389:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690304 00000100:00100000:1.0:1767318938.578394:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690304 00000100:00100000:1.0:1767318938.578439:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068 00000100:00100000:1.0:1767318938.578446:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982689792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318938.578467:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318938.578543:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982689792:12345-0@lo:38:kworker.0 Request processed in 96us (10024us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318938.578550:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068 00000100:00100000:1.0:1767318938.578565:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1069 00000100:00100000:1.0:1767318938.578569:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982690304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318938.578579:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318938.578607:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982690304:12345-0@lo:38:kworker.0 Request processed in 36us (9887us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318938.578612:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1069 00000100:00080000:3.0:1767318938.578697:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318938.578703:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318938.578706:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982689792:0@lo:38:kworker.0 00000100:00080000:3.0:1767318938.578715:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318938.578718:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318938.578721:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982690304:0@lo:38:kworker.0 00010000:00010000:1.0:1767318938.756694:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318939.780663:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318940.555492:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318940.555621:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597056 00000100:00100000:2.0:1767318940.555632:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597056 00000100:00100000:2.0:1767318940.555725:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 817 00000100:00100000:2.0:1767318940.555736:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915597056:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318940.555830:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915597056:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 96us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318940.555838:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 817 00000100:00080000:3.0:1767318940.555948:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318940.556175:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318940.556243:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915596928 00000100:00100000:2.0:1767318940.556248:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915596928 00000100:00100000:2.0:1767318940.556309:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1070 00000100:00100000:2.0:1767318940.556314:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915596928:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318940.556334:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318940.556377:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915596928:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (202us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318940.556384:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1070 00010000:00010000:1.0:1767318940.804863:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318941.828778:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318942.148664:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982690432:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318942.148733:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318942.150313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982690560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318942.150386:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318942.150519:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690432 00000100:00100000:3.0:1767318942.150527:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690432 00000100:00100000:3.0:1767318942.150593:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690560 00000100:00100000:3.0:1767318942.150597:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690560 00000100:00100000:3.0:1767318942.150619:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 564 00000100:00100000:3.0:1767318942.150627:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982690432:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318942.150634:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 565 00000100:00100000:0.0:1767318942.150645:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982690560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318942.150652:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:0.0:1767318942.150665:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318942.150695:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982690432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (1961us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318942.150701:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 564 00000100:00100000:0.0:1767318942.150717:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982690560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318942.150725:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 565 00000100:00100000:3.0:1767318942.150754:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982690432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318942.150768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982690560:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318942.150841:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318942.150846:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767318942.857529:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318943.684633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.684643:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318943.684698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.684702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.684707:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.684714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.684719:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.684725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318943.684744:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982690816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318943.684773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.684776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.684779:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.684782:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.684785:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.684788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318943.684799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982691072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318943.684807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.684810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.684811:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.684814:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.684816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.684819:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318943.684826:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982691200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318943.684866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318943.684869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318943.684871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318943.684874:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4187 00000100:00080000:2.0:1767318943.684876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318943.684879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318943.684889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0a80 x1853163982691584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318943.685037:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982691456:0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.685098:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685144:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767318943.685149:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982691328:0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.685169:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685189:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982690816:0@lo:38:kworker.0 00000100:00100000:0.0:1767318943.685210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685225:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982691072:0@lo:38:kworker.0 00000100:00100000:0.0:1767318943.685241:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685251:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982691200:0@lo:38:kworker.0 00000100:00100000:0.0:1767318943.685268:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982691584:0@lo:38:kworker.0 00000100:00100000:0.0:1767318943.685322:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318943.685347:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690816 00000100:00100000:0.0:1767318943.685354:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690816 00000100:00100000:0.0:1767318943.685393:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691072 00000100:00100000:0.0:1767318943.685395:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691072 00000100:00100000:0.0:1767318943.685404:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691584 00000100:00100000:0.0:1767318943.685406:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691584 00000100:00100000:0.0:1767318943.685417:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1767318943.685424:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982690816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.685443:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.685479:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982690816:12345-0@lo:38:kworker.0 Request processed in 55us (267us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.685486:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1767318943.685500:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1767318943.685503:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982691072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.685511:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.685527:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982691072:12345-0@lo:38:kworker.0 Request processed in 23us (286us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.685532:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1767318943.685541:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073 00000100:00100000:0.0:1767318943.685544:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982691584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318943.685552:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318943.685568:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982691584:12345-0@lo:38:kworker.0 Request processed in 23us (245us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318943.685573:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073 00000100:00080000:0.0:1767318943.685601:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318943.685605:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318943.685609:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982690816:0@lo:38:kworker.0 00000100:00080000:0.0:1767318943.685615:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318943.685618:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318943.685621:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982691072:0@lo:38:kworker.0 00000100:00080000:0.0:1767318943.685627:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318943.685630:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318943.685633:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982691584:0@lo:38:kworker.0 00000100:00100000:1.0:1767318943.685856:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982690944:0@lo:400:kworker.0 00000100:00100000:1.0:1767318943.685886:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318943.685924:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318943.685929:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982690688:0@lo:400:kworker.0 00000100:00100000:1.0:1767318943.685950:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318943.685972:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690944 00000100:00100000:1.0:1767318943.685976:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690944 00000100:00100000:1.0:1767318943.686010:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982690688 00000100:00100000:1.0:1767318943.686013:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982690688 00000100:00100000:1.0:1767318943.686032:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 818 00000100:00100000:1.0:1767318943.686038:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982690944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318943.686079:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982690944:12345-0@lo:400:kworker.0 Request processed in 41us (193us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318943.686085:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 818 00000100:00100000:1.0:1767318943.686098:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 819 00000100:00100000:1.0:1767318943.686103:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982690688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318943.686142:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982690688:12345-0@lo:400:kworker.0 Request processed in 40us (191us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318943.686151:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 819 00000100:00080000:1.0:1767318943.686167:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318943.686188:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580a80 x1853163982690688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318959 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318943.686198:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982690688:0@lo:400:kworker.0 00010000:00080000:1.0:1767318943.686207:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581880 x1853163982690944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318959 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318943.686214:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982690944:0@lo:400:kworker.0 00000100:00100000:3.0:1767318943.686346:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691456 00000100:00100000:3.0:1767318943.686351:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691456 00000100:00100000:3.0:1767318943.686393:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691328 00000100:00100000:3.0:1767318943.686397:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691328 00000100:00100000:3.0:1767318943.686416:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 535 00000100:00100000:3.0:1767318943.686421:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982691456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318943.686456:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982691456:12345-0@lo:400:kworker.0 Request processed in 34us (1357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318943.686465:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 535 00000100:00100000:3.0:1767318943.686477:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 536 00000100:00100000:3.0:1767318943.686482:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982691328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318943.686508:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982691328:12345-0@lo:400:kworker.0 Request processed in 26us (1338us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318943.686513:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 536 00000100:00100000:3.0:1767318943.686533:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691200 00000100:00100000:3.0:1767318943.686537:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691200 00000100:00100000:3.0:1767318943.686557:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 293 00000100:00100000:3.0:1767318943.686562:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982691200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318943.686574:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318943.686604:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982691200:12345-0@lo:38:kworker.0 Request processed in 40us (1334us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318943.686610:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 293 00000100:00100000:0.0:1767318943.687350:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982691328:0@lo:400:kworker.0 00000100:00100000:0.0:1767318943.687374:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982691456:0@lo:400:kworker.0 00000100:00080000:0.0:1767318943.687409:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318943.687413:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318943.687417:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982691200:0@lo:38:kworker.0 00010000:00010000:1.0:1767318943.877787:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318944.900790:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318945.676672:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318945.677265:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597312 00000100:00100000:2.0:1767318945.677309:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597312 00000100:00100000:2.0:1767318945.677405:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 820 00000100:00100000:2.0:1767318945.677417:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915597312:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318945.677525:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915597312:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 111us (868us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318945.677533:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 820 00000100:00080000:1.0:1767318945.677610:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318945.679383:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318945.679953:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597184 00000100:00100000:2.0:1767318945.679964:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597184 00000100:00100000:2.0:1767318945.680018:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1074 00000100:00100000:2.0:1767318945.680026:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915597184:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318945.680053:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318945.680173:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915597184:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 145us (798us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318945.680180:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1074 00010000:00010000:2.0:1767318945.924684:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318946.948776:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318947.268612:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318947.268645:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982691712:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318947.268716:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318947.268911:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691712 00000100:00100000:1.0:1767318947.268921:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691712 00000100:00100000:1.0:1767318947.268994:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 566 00000100:00100000:1.0:1767318947.269003:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982691712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318947.269033:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318947.269084:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982691712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (367us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318947.269091:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 566 00000100:00100000:1.0:1767318947.269143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982691712:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318947.269376:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767318947.274076:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982691840:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318947.274136:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318947.274236:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691840 00000100:00100000:2.0:1767318947.274243:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691840 00000100:00100000:2.0:1767318947.274325:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 567 00000100:00100000:2.0:1767318947.274335:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982691840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318947.274360:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767318947.276312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318947.280187:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982691840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5840us (6036us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318947.280204:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 567 00000100:00100000:1.0:1767318947.280352:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982691840:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767318947.974082:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767318948.804418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318948.804429:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767318948.804495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318948.804500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318948.804504:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318948.804512:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4192 00000100:00080000:3.0:1767318948.804516:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318948.804523:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318948.804545:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163982692096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318948.804581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318948.804583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318948.804585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318948.804589:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4192 00000100:00080000:3.0:1767318948.804591:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318948.804594:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318948.804605:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246d80 x1853163982692352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318948.804612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318948.804615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318948.804617:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318948.804620:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4192 00000100:00080000:3.0:1767318948.804622:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318948.804625:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318948.804638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e244000 x1853163982692480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318948.804681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318948.804684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318948.804687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318948.804690:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4192 00000100:00080000:3.0:1767318948.804692:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318948.804695:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318948.804706:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0a80 x1853163982692864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767318948.804738:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982692608:0@lo:400:kworker.0 00000100:00100000:3.0:1767318948.804849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318948.804888:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767318948.804894:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982692736:0@lo:400:kworker.0 00000100:00100000:3.0:1767318948.804917:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805041:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982692096:0@lo:38:kworker.0 00000100:00100000:2.0:1767318948.805081:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805114:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982692352:0@lo:38:kworker.0 00000100:00100000:2.0:1767318948.805135:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805152:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982692480:0@lo:38:kworker.0 00000100:00100000:2.0:1767318948.805173:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805209:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982692864:0@lo:38:kworker.0 00000100:00100000:2.0:1767318948.805230:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805263:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692608 00000100:00100000:2.0:1767318948.805292:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692608 00000100:00100000:2.0:1767318948.805333:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692736 00000100:00100000:2.0:1767318948.805336:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692736 00000100:00100000:2.0:1767318948.805355:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 537 00000100:00100000:2.0:1767318948.805363:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982692608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318948.805383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982692224:0@lo:400:kworker.0 00000100:00100000:2.0:1767318948.805413:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982692608:12345-0@lo:400:kworker.0 Request processed in 50us (564us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318948.805420:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 537 00000100:00100000:1.0:1767318948.805422:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805436:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 538 00000100:00100000:1.0:1767318948.805442:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318948.805442:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982692736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318948.805446:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982691968:0@lo:400:kworker.0 00000100:00100000:1.0:1767318948.805466:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318948.805468:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982692736:12345-0@lo:400:kworker.0 Request processed in 27us (551us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318948.805473:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 538 00000100:00100000:2.0:1767318948.805490:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692096 00000100:00100000:1.0:1767318948.805491:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692352 00000100:00100000:2.0:1767318948.805493:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692096 00000100:00100000:1.0:1767318948.805494:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692352 00000100:00100000:1.0:1767318948.805504:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692224 00000100:00100000:2.0:1767318948.805504:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692864 00000100:00100000:1.0:1767318948.805507:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692224 00000100:00100000:2.0:1767318948.805507:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692864 00000100:00100000:2.0:1767318948.805513:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982691968 00000100:00100000:2.0:1767318948.805515:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982691968 00000100:00100000:2.0:1767318948.805591:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 821 00000100:00100000:2.0:1767318948.805601:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982691968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318948.805643:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982691968:12345-0@lo:400:kworker.0 Request processed in 44us (176us total) trans 0 rc 0/0 00000100:00080000:0.0:1767318948.805658:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318948.805704:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0380 x1853163982691968/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318964 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318948.805720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982691968:0@lo:400:kworker.0 00000100:00100000:3.0:1767318948.811251:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1075 00000100:00100000:3.0:1767318948.811265:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318948.811294:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318948.811381:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692096:12345-0@lo:38:kworker.0 Request processed in 119us (6302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318948.811387:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1075 00000100:00100000:3.0:1767318948.811413:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076 00000100:00100000:3.0:1767318948.811416:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318948.811423:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318948.811643:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692352:12345-0@lo:38:kworker.0 Request processed in 224us (6507us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318948.811648:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076 00000100:00100000:3.0:1767318948.811660:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077 00000100:00100000:3.0:1767318948.811663:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318948.811672:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318948.811695:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982692864:12345-0@lo:38:kworker.0 Request processed in 31us (6464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318948.811701:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077 00000100:00080000:3.0:1767318948.811759:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318948.811765:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318948.811768:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982692096:0@lo:38:kworker.0 00000100:00080000:3.0:1767318948.811779:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318948.811782:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318948.811785:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982692352:0@lo:38:kworker.0 00000100:00080000:3.0:1767318948.811794:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318948.811797:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318948.811886:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982692864:0@lo:38:kworker.0 00000100:00100000:3.0:1767318948.811951:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982692736:0@lo:400:kworker.0 00000100:00100000:3.0:1767318948.811960:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982692608:0@lo:400:kworker.0 00000100:00100000:3.0:1767318948.811997:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692480 00000100:00100000:3.0:1767318948.812002:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692480 00000100:00100000:3.0:1767318948.812022:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 294 00000100:00100000:3.0:1767318948.812025:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982692480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318948.812033:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318948.812061:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982692480:12345-0@lo:38:kworker.0 Request processed in 34us (6887us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318948.812067:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 294 00000100:00080000:3.0:1767318948.812092:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318948.812096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318948.812098:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982692480:0@lo:38:kworker.0 00000100:00100000:2.0:1767318948.813566:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 821 00000100:00100000:1.0:1767318948.813844:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 822 00000100:00100000:1.0:1767318948.813861:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982692224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318948.818336:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982692224:12345-0@lo:400:kworker.0 Request processed in 4472us (12901us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318948.818352:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 822 00010000:00080000:0.0:1767318948.821209:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3b80 x1853163982692224/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318964 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318948.821232:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982692224:0@lo:400:kworker.0 00010000:00010000:2.0:1767318948.998597:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318950.022160:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318950.796448:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318950.796497:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318950.796595:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597440 00000100:00100000:0.0:1767318950.796606:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597440 00000100:00100000:0.0:1767318950.796652:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597568 00000100:00100000:0.0:1767318950.796656:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597568 00000100:00100000:0.0:1767318950.796709:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 823 00000100:00100000:0.0:1767318950.796720:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915597568:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318950.796824:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915597568:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 105us (324us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318950.796832:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 823 00000100:00100000:0.0:1767318950.796850:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1078 00000100:00100000:0.0:1767318950.796854:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915597440:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318950.796873:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318950.796898:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915597440:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (461us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318950.796905:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1078 00000100:00080000:3.0:1767318950.797324:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318951.044546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318952.068513:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318952.388946:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318952.388975:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982692992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318952.389063:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318952.389315:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982692992 00000100:00100000:1.0:1767318952.389324:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982692992 00000100:00100000:1.0:1767318952.389391:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 568 00000100:00100000:1.0:1767318952.389400:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982692992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318952.389429:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318952.389479:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982692992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (415us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318952.389486:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 568 00000100:00100000:1.0:1767318952.389544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982692992:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318952.389800:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767318952.390451:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318952.390464:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982693120:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318952.390517:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318952.390731:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693120 00000100:00100000:0.0:1767318952.390755:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693120 00000100:00100000:0.0:1767318952.390804:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 569 00000100:00100000:0.0:1767318952.390813:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982693120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318952.390834:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318952.390869:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982693120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318952.390876:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 569 00000100:00100000:3.0:1767318952.392450:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982693120:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318952.392989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767318953.094108:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767318953.924626:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318953.924636:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767318953.924702:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318953.924706:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318953.924710:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318953.924727:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4197 00000100:00080000:1.0:1767318953.924732:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318953.924739:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318953.924768:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163982693376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318953.924827:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318953.924830:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318953.924832:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318953.924836:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4197 00000100:00080000:1.0:1767318953.924838:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318953.924843:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318953.924855:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163982693632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318953.924864:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318953.924867:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318953.924880:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318953.924884:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4197 00000100:00080000:1.0:1767318953.924886:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318953.924890:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318953.924903:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd500 x1853163982693760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318953.924934:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318953.924938:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318953.924940:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318953.924943:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4197 00000100:00080000:1.0:1767318953.924945:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318953.924948:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318953.924966:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982694144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318953.925001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982694016:0@lo:400:kworker.0 00000100:00100000:1.0:1767318953.925070:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318953.925130:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767318953.925136:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982693888:0@lo:400:kworker.0 00000100:00100000:1.0:1767318953.925159:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318953.925620:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982693376:0@lo:38:kworker.0 00000100:00100000:2.0:1767318953.925669:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318953.925698:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982693632:0@lo:38:kworker.0 00000100:00100000:2.0:1767318953.925758:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318953.925777:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982693760:0@lo:38:kworker.0 00000100:00100000:2.0:1767318953.925818:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318953.925840:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982694144:0@lo:38:kworker.0 00000100:00100000:2.0:1767318953.925859:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318953.925888:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693376 00000100:00100000:2.0:1767318953.925896:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693376 00000100:00100000:2.0:1767318953.925934:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693632 00000100:00100000:2.0:1767318953.925937:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693632 00000100:00100000:2.0:1767318953.925949:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694144 00000100:00100000:2.0:1767318953.925952:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694144 00000100:00100000:2.0:1767318953.925963:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079 00000100:00100000:2.0:1767318953.925970:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982693376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318953.925988:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318953.926027:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982693376:12345-0@lo:38:kworker.0 Request processed in 56us (357us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318953.926034:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079 00000100:00100000:2.0:1767318953.926050:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080 00000100:00100000:2.0:1767318953.926054:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982693632:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318953.926063:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318953.926099:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982693632:12345-0@lo:38:kworker.0 Request processed in 43us (378us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318953.926104:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080 00000100:00100000:2.0:1767318953.926114:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1081 00000100:00100000:2.0:1767318953.926117:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982694144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318953.926125:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318953.926145:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982694144:12345-0@lo:38:kworker.0 Request processed in 26us (284us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318953.926150:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1081 00000100:00080000:2.0:1767318953.926185:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318953.926190:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318953.926194:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982693376:0@lo:38:kworker.0 00000100:00080000:2.0:1767318953.926203:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318953.926206:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318953.926208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982693632:0@lo:38:kworker.0 00000100:00080000:2.0:1767318953.926216:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318953.926218:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318953.926220:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982694144:0@lo:38:kworker.0 00000100:00100000:0.0:1767318953.926810:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982693504:0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.926849:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318953.926883:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318953.926889:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982693248:0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.926909:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318953.926949:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694016 00000100:00100000:0.0:1767318953.926955:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694016 00000100:00100000:0.0:1767318953.926990:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693888 00000100:00100000:0.0:1767318953.926994:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693888 00000100:00100000:0.0:1767318953.927015:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 539 00000100:00100000:0.0:1767318953.927023:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982694016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.927318:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982694016:12345-0@lo:400:kworker.0 Request processed in 296us (2247us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318953.927326:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 539 00000100:00100000:0.0:1767318953.927342:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 540 00000100:00100000:0.0:1767318953.927349:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982693888:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.927381:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982693888:12345-0@lo:400:kworker.0 Request processed in 32us (2220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318953.927386:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 540 00000100:00100000:0.0:1767318953.927404:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693504 00000100:00100000:0.0:1767318953.927408:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693504 00000100:00100000:0.0:1767318953.927436:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693248 00000100:00100000:0.0:1767318953.927439:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693248 00000100:00100000:0.0:1767318953.927453:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 824 00000100:00100000:0.0:1767318953.927459:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982693504:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.927493:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982693504:12345-0@lo:400:kworker.0 Request processed in 33us (643us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318953.927500:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 824 00000100:00100000:0.0:1767318953.927511:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 825 00000100:00100000:0.0:1767318953.927517:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982693248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.927543:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982693248:12345-0@lo:400:kworker.0 Request processed in 27us (633us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318953.927549:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 825 00000100:00080000:0.0:1767318953.927572:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318953.927602:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982693888:0@lo:400:kworker.0 00000100:00100000:0.0:1767318953.927614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982694016:0@lo:400:kworker.0 00010000:00080000:0.0:1767318953.927753:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1500 x1853163982693248/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318969 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318953.927766:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982693248:0@lo:400:kworker.0 00010000:00080000:0.0:1767318953.927776:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382de300 x1853163982693504/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318969 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318953.927862:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982693504:0@lo:400:kworker.0 00000100:00100000:3.0:1767318953.929396:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982693760 00000100:00100000:3.0:1767318953.929407:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982693760 00000100:00100000:3.0:1767318953.929468:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 295 00000100:00100000:3.0:1767318953.929475:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982693760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318953.929497:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318953.929579:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982693760:12345-0@lo:38:kworker.0 Request processed in 102us (3759us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318953.929586:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 295 00000100:00080000:2.0:1767318953.929819:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767318953.929826:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767318953.929830:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982693760:0@lo:38:kworker.0 00010000:00010000:1.0:1767318954.118053:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318955.140519:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318955.916754:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318955.916847:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318955.916977:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597824 00000100:00100000:3.0:1767318955.916988:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597824 00000100:00100000:0.0:1767318955.917039:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915597696 00000100:00100000:0.0:1767318955.917045:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915597696 00000100:00100000:0.0:1767318955.917093:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1082 00000100:00100000:0.0:1767318955.917102:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915597696:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318955.917125:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318955.917194:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915597696:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (346us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318955.917202:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1082 00000100:00100000:3.0:1767318955.917440:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 826 00000100:00100000:3.0:1767318955.917449:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915597824:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767318955.917493:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915597824:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 46us (752us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318955.917500:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 826 00000100:00080000:2.0:1767318955.927145:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318956.164714:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318957.191377:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318957.508941:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982694272:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318957.509028:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318957.509260:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694272 00000100:00100000:3.0:1767318957.509268:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694272 00000100:00100000:3.0:1767318957.509352:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 570 00000100:00100000:3.0:1767318957.509360:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982694272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318957.509391:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318957.509437:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982694272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (409us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318957.509445:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 570 00000100:00100000:3.0:1767318957.509500:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982694272:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318957.510075:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767318957.511615:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982694400:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318957.511686:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318957.511820:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694400 00000100:00100000:3.0:1767318957.511827:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694400 00000100:00100000:3.0:1767318957.512018:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 571 00000100:00100000:3.0:1767318957.512029:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982694400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318957.512058:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318957.512106:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982694400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318957.512113:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 571 00000100:00080000:0.0:1767318957.512160:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318957.512396:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982694400:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767318958.213198:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767318959.044499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318959.044510:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318959.044582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318959.044587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318959.044591:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318959.044600:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4202 00000100:00080000:2.0:1767318959.044605:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318959.044612:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318959.044647:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163982694656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318959.044682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318959.044685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318959.044687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318959.044691:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4202 00000100:00080000:2.0:1767318959.044694:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318959.044698:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318959.044708:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594380 x1853163982694912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318959.044717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318959.044720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318959.044722:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318959.044725:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4202 00000100:00080000:2.0:1767318959.044727:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318959.044731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318959.044741:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594000 x1853163982695040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318959.044786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318959.044790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318959.044791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318959.044795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4202 00000100:00080000:2.0:1767318959.044797:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318959.044801:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318959.044837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988e00 x1853163982695424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318959.044878:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982695296:0@lo:400:kworker.0 00000100:00100000:2.0:1767318959.044943:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318959.045006:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318959.045012:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982695168:0@lo:400:kworker.0 00000100:00100000:2.0:1767318959.045034:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.045124:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982694656:0@lo:38:kworker.0 00000100:00100000:1.0:1767318959.045156:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.045201:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982694912:0@lo:38:kworker.0 00000100:00100000:1.0:1767318959.045233:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.045250:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982695040:0@lo:38:kworker.0 00000100:00100000:1.0:1767318959.045290:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.045315:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982695424:0@lo:38:kworker.0 00000100:00100000:1.0:1767318959.045336:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318959.045369:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695040 00000100:00100000:1.0:1767318959.045377:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695040 00000100:00100000:1.0:1767318959.045414:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 296 00000100:00100000:1.0:1767318959.045421:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982695040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318959.045445:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318959.045494:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982695040:12345-0@lo:38:kworker.0 Request processed in 72us (221us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318959.045502:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 296 00000100:00080000:1.0:1767318959.045547:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318959.045552:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318959.045556:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982695040:0@lo:38:kworker.0 00000100:00100000:3.0:1767318959.045907:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982694528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.045940:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318959.045962:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767318959.045968:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982694784:0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.045989:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318959.046014:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695296 00000100:00100000:3.0:1767318959.046020:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695296 00000100:00100000:3.0:1767318959.046058:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695168 00000100:00100000:3.0:1767318959.046061:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695168 00000100:00100000:3.0:1767318959.046080:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 541 00000100:00100000:3.0:1767318959.046086:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982695296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046131:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982695296:12345-0@lo:400:kworker.0 Request processed in 44us (1187us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318959.046141:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 541 00000100:00100000:3.0:1767318959.046155:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 542 00000100:00100000:3.0:1767318959.046162:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982695168:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046188:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982695168:12345-0@lo:400:kworker.0 Request processed in 27us (1153us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318959.046194:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 542 00000100:00100000:3.0:1767318959.046211:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694656 00000100:00100000:3.0:1767318959.046214:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694656 00000100:00100000:3.0:1767318959.046228:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694912 00000100:00100000:3.0:1767318959.046231:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694912 00000100:00100000:3.0:1767318959.046238:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695424 00000100:00100000:3.0:1767318959.046240:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695424 00000100:00100000:3.0:1767318959.046246:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694528 00000100:00100000:3.0:1767318959.046248:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694528 00000100:00100000:3.0:1767318959.046265:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982694784 00000100:00100000:3.0:1767318959.046289:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982694784 00000100:00100000:3.0:1767318959.046307:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827 00000100:00100000:3.0:1767318959.046312:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982694528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046347:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982694528:12345-0@lo:400:kworker.0 Request processed in 35us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318959.046353:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827 00000100:00100000:3.0:1767318959.046366:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 828 00000100:00100000:3.0:1767318959.046372:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982694784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046397:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982694784:12345-0@lo:400:kworker.0 Request processed in 26us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318959.046404:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 828 00000100:00100000:3.0:1767318959.046415:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1767318959.046420:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982694656:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318959.046434:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318959.046460:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982694656:12345-0@lo:38:kworker.0 Request processed in 38us (1303us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318959.046466:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1767318959.046477:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084 00000100:00100000:3.0:1767318959.046481:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982694912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318959.046489:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318959.046512:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982694912:12345-0@lo:38:kworker.0 Request processed in 30us (1279us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318959.046518:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084 00000100:00100000:3.0:1767318959.046530:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085 00000100:00100000:3.0:1767318959.046534:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982695424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318959.046543:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318959.046564:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982695424:12345-0@lo:38:kworker.0 Request processed in 29us (1228us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318959.046570:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085 00010000:00080000:3.0:1767318959.046607:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597800 x1853163982694784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318975 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318959.046620:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982694784:0@lo:400:kworker.0 00010000:00080000:3.0:1767318959.046646:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596a00 x1853163982694528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318975 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318959.046655:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982694528:0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046683:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982695168:0@lo:400:kworker.0 00000100:00100000:3.0:1767318959.046694:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982695296:0@lo:400:kworker.0 00000100:00080000:0.0:1767318959.046703:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318959.046708:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767318959.046710:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767318959.046783:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318959.046788:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318959.046791:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982694656:0@lo:38:kworker.0 00000100:00080000:1.0:1767318959.046799:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318959.046802:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318959.046805:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982694912:0@lo:38:kworker.0 00000100:00080000:1.0:1767318959.046812:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318959.046815:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318959.046818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982695424:0@lo:38:kworker.0 00010000:00010000:1.0:1767318959.240023:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318960.017904:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767318960.018001:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915597952 offset 0 length 520: 4 00000100:00100000:3.0:1767318960.018023:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318960.029974:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598208 00000100:00100000:2.0:1767318960.030031:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598208 00000100:00100000:2.0:1767318960.030808:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598080 00000100:00100000:2.0:1767318960.030815:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598080 00000100:00100000:2.0:1767318960.030851:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 829 00000100:00100000:2.0:1767318960.030863:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915598208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318960.030964:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915598208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (13071us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318960.030973:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 829 00000100:00100000:2.0:1767318960.031008:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1086 00000100:00100000:2.0:1767318960.031012:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915598080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318960.031033:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318960.031077:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915598080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (13053us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318960.031084:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1086 00000100:00080000:0.0:1767318960.035693:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318960.260531:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767318961.285405:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318962.309957:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318962.628724:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318962.635810:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982695680:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318962.635900:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318962.635992:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982695552:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318962.636018:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318962.637746:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695680 00000100:00100000:1.0:1767318962.637760:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695680 00000100:00100000:1.0:1767318962.637845:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695552 00000100:00100000:1.0:1767318962.637849:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695552 00000100:00100000:1.0:1767318962.637872:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 572 00000100:00100000:1.0:1767318962.637884:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982695680:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767318962.637924:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318962.638017:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982695680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 135us (2116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318962.638024:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 572 00000100:00100000:1.0:1767318962.638045:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 573 00000100:00100000:1.0:1767318962.638052:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982695552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318962.638069:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318962.638100:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982695552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (2081us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318962.638105:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 573 00000100:00100000:1.0:1767318962.638175:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982695680:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318962.638191:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982695552:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318962.638528:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318962.638533:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318963.143476:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318938/real 1767318938] req@ffff93463f963100 x1853163982689280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318963 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318963.143521:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463f963100 x1853163982689280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318963 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767318963.143537:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318963.143562:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767318963.143572:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982689280:0@lo:250:kworker.0 00000100:00080000:0.0:1767318963.144742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318963.144750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767318963.144753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318963.144759:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4187 00000100:00080000:0.0:1767318963.144763:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318963.144770:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318963.144805:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0380 x1853163982695808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318963.144871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318963.144876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318963.144877:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318963.144881:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4207 00000100:00080000:0.0:1767318963.144883:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318963.144887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318963.144903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581880 x1853163982696064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318963.144908:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767318963.144924:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982695936:0@lo:400:kworker.0 00000100:00080000:0.0:1767318963.144935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318963.144938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318963.144940:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318963.144943:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4207 00000100:00080000:0.0:1767318963.144945:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318963.144948:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318963.144958:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982696320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318963.144965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318963.144968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318963.144969:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318963.144980:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4207 00000100:00080000:0.0:1767318963.144982:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318963.144985:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318963.144989:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.144996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163982696448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318963.145004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318963.145008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318963.145009:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318963.145012:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4207 00000100:00080000:0.0:1767318963.145014:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318963.145017:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318963.145028:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627800 x1853163982696576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318963.145048:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982696192:0@lo:400:kworker.0 00000100:00100000:2.0:1767318963.145070:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.145262:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982695936 00000100:00100000:0.0:1767318963.145301:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982695936 00000100:00100000:0.0:1767318963.145359:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696192 00000100:00100000:0.0:1767318963.145363:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696192 00000100:00100000:0.0:1767318963.145384:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830 00000100:00100000:0.0:1767318963.145392:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982695936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318963.145442:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982695936:12345-0@lo:400:kworker.0 Request processed in 50us (453us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318963.145450:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830 00000100:00100000:0.0:1767318963.145464:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767318963.145469:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982696192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318963.145583:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982696192:12345-0@lo:400:kworker.0 Request processed in 113us (510us total) trans 0 rc 0/0 00000100:00080000:2.0:1767318963.145583:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318963.145589:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831 00010000:00080000:0.0:1767318963.145624:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580a80 x1853163982696192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318979 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318963.145637:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982696192:0@lo:400:kworker.0 00010000:00080000:0.0:1767318963.145648:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3b80 x1853163982695936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318979 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318963.145656:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982695936:0@lo:400:kworker.0 00000100:00100000:3.0:1767318963.146069:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982695808:0@lo:250:kworker.0 00000400:00000100:3.0:1767318963.146097:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982695808 offset 0 length 520: 4 00000100:00100000:3.0:1767318963.146119:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982696064:0@lo:38:kworker.0 00000100:00100000:3.0:1767318963.146138:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318963.146265:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696064 00000100:00100000:1.0:1767318963.146290:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696064 00000100:00100000:1.0:1767318963.146329:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1087 00000100:00100000:1.0:1767318963.146335:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318963.146355:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318963.146389:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696064:12345-0@lo:38:kworker.0 Request processed in 54us (251us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318963.146395:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1087 00000100:00100000:3.0:1767318963.146487:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982696320:0@lo:38:kworker.0 00000100:00100000:3.0:1767318963.146526:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318963.146594:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982696448:0@lo:38:kworker.0 00000100:00100000:3.0:1767318963.146620:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318963.146649:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982696576:0@lo:38:kworker.0 00000100:00100000:3.0:1767318963.146670:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318963.146714:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696448 00000100:00100000:0.0:1767318963.146719:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696448 00000100:00100000:0.0:1767318963.146747:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 297 00000100:00100000:0.0:1767318963.146751:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982696448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318963.146765:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318963.146792:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982696448:12345-0@lo:38:kworker.0 Request processed in 40us (172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318963.146797:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 297 00000100:00080000:3.0:1767318963.148879:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318963.148894:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318963.148901:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982696064:0@lo:38:kworker.0 00000100:00080000:3.0:1767318963.148914:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318963.148917:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318963.148919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982696448:0@lo:38:kworker.0 00000100:00100000:2.0:1767318963.152072:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696320 00000100:00100000:2.0:1767318963.152090:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696320 00000100:00100000:2.0:1767318963.152216:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696576 00000100:00100000:2.0:1767318963.152221:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696576 00000100:00100000:1.0:1767318963.152310:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088 00000100:00100000:1.0:1767318963.152320:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318963.152344:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318963.152444:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696320:12345-0@lo:38:kworker.0 Request processed in 122us (5915us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318963.152451:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088 00000100:00080000:3.0:1767318963.152463:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767318963.152467:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089 00000100:00080000:3.0:1767318963.152468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318963.152471:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696576:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318963.152472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982696320:0@lo:38:kworker.0 00010000:00020000:1.0:1767318963.152480:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318963.152863:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982696576:12345-0@lo:38:kworker.0 Request processed in 388us (6190us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318963.152871:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089 00000100:00080000:3.0:1767318963.153122:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318963.153128:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318963.153132:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982696576:0@lo:38:kworker.0 00010000:00010000:1.0:1767318963.332706:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318964.358108:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318965.132419:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318965.132478:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318965.132584:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598336 00000100:00100000:2.0:1767318965.132596:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598336 00000100:00100000:2.0:1767318965.132643:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598464 00000100:00100000:2.0:1767318965.132647:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598464 00000100:00100000:2.0:1767318965.132717:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 832 00000100:00100000:2.0:1767318965.132727:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915598464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318965.132809:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915598464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (329us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318965.132816:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 832 00000100:00100000:2.0:1767318965.132833:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1090 00000100:00100000:2.0:1767318965.132837:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915598336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318965.132861:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318965.132889:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915598336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 50us (480us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318965.132896:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1090 00000100:00080000:3.0:1767318965.133307:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318965.380498:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318966.404551:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318967.429268:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318967.748597:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1767318967.748635:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982696704:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767318967.748704:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318967.748745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982696832:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318967.748771:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318967.749124:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696704 00000100:00100000:0.0:1767318967.749132:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696704 00000100:00100000:0.0:1767318967.749195:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696832 00000100:00100000:0.0:1767318967.749198:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696832 00000100:00100000:0.0:1767318967.749217:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 574 00000100:00100000:0.0:1767318967.749227:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982696704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318967.749257:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318967.749333:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982696704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (627us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318967.749342:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 574 00000100:00100000:3.0:1767318967.749350:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 575 00000100:00100000:3.0:1767318967.749362:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982696832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767318967.749383:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318967.749392:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982696704:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318967.749453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318967.749457:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318967.756793:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982696832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 7422us (8008us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318967.756811:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 575 00000100:00100000:0.0:1767318967.763832:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982696832:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767318968.260359:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.260512:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318968.260596:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982696960:0@lo:400:kworker.0 00000100:00100000:2.0:1767318968.260666:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767318968.260794:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.260798:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.260801:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.260807:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.260810:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.260817:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318968.260857:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163982697088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.260893:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.260897:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.260898:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.260902:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.260904:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.260908:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318968.260920:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982697344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.260929:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.260932:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.260934:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.260937:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.260939:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.260942:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318968.260955:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581880 x1853163982697472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318968.260965:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318968.260968:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318968.260970:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318968.260973:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4211 00000100:00080000:0.0:1767318968.260975:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318968.260979:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318968.260990:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163982697600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318968.261011:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318968.261024:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982697216:0@lo:400:kworker.0 00000100:00100000:0.0:1767318968.261052:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.261167:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982697088:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.262473:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318968.262625:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982696960 00000100:00100000:1.0:1767318968.262634:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982696960 00000100:00100000:1.0:1767318968.262689:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697216 00000100:00100000:1.0:1767318968.262692:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697216 00000100:00100000:1.0:1767318968.262715:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 833 00000100:00100000:1.0:1767318968.262724:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982696960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318968.262801:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982696960:12345-0@lo:400:kworker.0 Request processed in 78us (2134us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318968.262810:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 833 00000100:00100000:1.0:1767318968.262826:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 834 00000100:00100000:1.0:1767318968.262832:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982697216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318968.263009:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982697216:12345-0@lo:400:kworker.0 Request processed in 178us (1957us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318968.263015:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 834 00010000:00080000:1.0:1767318968.263047:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3b80 x1853163982697216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318984 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318968.263059:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982697216:0@lo:400:kworker.0 00000100:00100000:3.0:1767318968.263221:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982697344:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.263247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.263262:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982697472:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.263295:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.263310:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982697600:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.263327:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318968.263350:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697472 00000100:00100000:3.0:1767318968.263354:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697472 00000100:00100000:3.0:1767318968.263461:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 298 00000100:00100000:3.0:1767318968.263466:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982697472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318968.263485:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318968.263519:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982697472:12345-0@lo:38:kworker.0 Request processed in 51us (222us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318968.263525:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 298 00000100:00080000:3.0:1767318968.263550:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.263555:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.263558:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982697472:0@lo:38:kworker.0 00000100:00100000:3.0:1767318968.263573:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697088 00000100:00100000:3.0:1767318968.263576:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697088 00000100:00100000:3.0:1767318968.263591:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697344 00000100:00100000:3.0:1767318968.263593:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697344 00000100:00100000:3.0:1767318968.263599:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697600 00000100:00100000:3.0:1767318968.263601:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697600 00000100:00100000:3.0:1767318968.263610:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091 00000100:00100000:3.0:1767318968.263614:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318968.263622:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318968.263648:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697088:12345-0@lo:38:kworker.0 Request processed in 34us (1180us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318968.263654:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091 00000100:00100000:3.0:1767318968.263665:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092 00000100:00100000:3.0:1767318968.263669:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318968.263680:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318968.263700:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697344:12345-0@lo:38:kworker.0 Request processed in 29us (452us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318968.263705:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092 00000100:00100000:3.0:1767318968.263715:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1093 00000100:00100000:3.0:1767318968.263719:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318968.263727:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318968.263748:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982697600:12345-0@lo:38:kworker.0 Request processed in 28us (420us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318968.263754:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1093 00000100:00080000:3.0:1767318968.263940:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.263945:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.263949:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982697088:0@lo:38:kworker.0 00000100:00080000:3.0:1767318968.263957:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.263960:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.263962:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982697344:0@lo:38:kworker.0 00000100:00080000:3.0:1767318968.263969:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318968.263971:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318968.263974:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982697600:0@lo:38:kworker.0 00010000:00080000:3.0:1767318968.264013:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c627800 x1853163982696960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318984 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318968.264024:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982696960:0@lo:400:kworker.0 00000100:00080000:0.0:1767318968.264163:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318968.454184:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318969.476527:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318970.256366:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318970.256567:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318970.259656:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598720 00000100:00100000:0.0:1767318970.259669:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598720 00000100:00080000:2.0:1767318970.260074:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767318970.260258:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598592 00000100:00100000:0.0:1767318970.260263:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598592 00000100:00100000:1.0:1767318970.260345:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 835 00000100:00100000:1.0:1767318970.260358:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915598720:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318970.260469:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915598720:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 114us (4111us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318970.260476:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 835 00000100:00100000:1.0:1767318970.260504:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1094 00000100:00100000:1.0:1767318970.260509:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915598592:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318970.260528:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318970.260554:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915598592:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (3987us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318970.260560:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1094 00010000:00010000:1.0:1767318970.500586:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318971.525922:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318972.549603:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318972.868515:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982697728:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318972.868591:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318972.868732:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318972.868744:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982697856:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318972.868786:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318972.869136:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697728 00000100:00100000:3.0:1767318972.869146:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697728 00000100:00100000:3.0:1767318972.869216:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697856 00000100:00100000:3.0:1767318972.869219:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697856 00000100:00100000:3.0:1767318972.869238:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 576 00000100:00100000:3.0:1767318972.869247:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982697728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767318972.869304:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318972.869357:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 577 00000100:00100000:3.0:1767318972.869357:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982697728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 111us (765us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318972.869375:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 576 00000100:00100000:1.0:1767318972.869376:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982697856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767318972.869391:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767318972.869430:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982697856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318972.869447:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 577 00000100:00100000:1.0:1767318972.869493:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982697728:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767318972.869591:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767318972.869595:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767318972.869649:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982697856:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767318973.380484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.380496:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767318973.380557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.380561:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.380564:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767318973.380566:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767318973.380572:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.380576:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767318973.380580:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982697984:0@lo:400:kworker.0 00000100:00080000:2.0:1767318973.380582:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318973.380601:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982698112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318973.380636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.380640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.380641:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.380645:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.380647:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.380650:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318973.380659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989c00 x1853163982698368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318973.380664:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767318973.380668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318973.380671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.380673:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.380676:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.380678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.380681:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318973.380691:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163982698496/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318973.380706:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767318973.380711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767318973.380713:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982698240:0@lo:400:kworker.0 00000100:00080000:2.0:1767318973.380714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318973.380716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318973.380719:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4216 00000100:00080000:2.0:1767318973.380721:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318973.380725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318973.380735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988e00 x1853163982698624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318973.380736:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318973.380766:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982697984 00000100:00100000:2.0:1767318973.380773:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982697984 00000100:00100000:1.0:1767318973.380776:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698240 00000100:00100000:1.0:1767318973.380778:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698240 00000100:00100000:2.0:1767318973.380823:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836 00000100:00100000:2.0:1767318973.380833:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982697984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767318973.380841:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982698112:0@lo:38:kworker.0 00000100:00100000:3.0:1767318973.380870:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318973.380881:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982697984:12345-0@lo:400:kworker.0 Request processed in 50us (217us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318973.380885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982698368:0@lo:38:kworker.0 00000100:00100000:2.0:1767318973.380888:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836 00000100:00100000:2.0:1767318973.380902:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698112 00000100:00100000:3.0:1767318973.380903:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318973.380905:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698112 00000100:00100000:2.0:1767318973.380914:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698368 00000100:00100000:2.0:1767318973.380916:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698368 00000100:00100000:3.0:1767318973.380916:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982698496:0@lo:38:kworker.0 00000100:00100000:2.0:1767318973.380924:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095 00000100:00100000:2.0:1767318973.380928:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698112:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767318973.380935:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767318973.380946:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318973.380965:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982698624:0@lo:38:kworker.0 00000100:00100000:2.0:1767318973.380970:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698112:12345-0@lo:38:kworker.0 Request processed in 41us (100us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318973.380975:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767318973.380982:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318973.380984:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698624 00000100:00100000:2.0:1767318973.380987:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698624 00000100:00100000:2.0:1767318973.380996:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096 00000100:00100000:2.0:1767318973.380999:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318973.381007:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767318973.381007:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318973.381011:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318973.381014:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982698112:0@lo:38:kworker.0 00000100:00100000:2.0:1767318973.381025:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698368:12345-0@lo:38:kworker.0 Request processed in 25us (122us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318973.381030:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096 00000100:00080000:3.0:1767318973.381032:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318973.381035:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318973.381038:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982698368:0@lo:38:kworker.0 00000100:00100000:2.0:1767318973.381039:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097 00000100:00100000:2.0:1767318973.381043:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318973.381051:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318973.381060:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698496 00000100:00100000:3.0:1767318973.381063:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698496 00000100:00100000:2.0:1767318973.381074:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982698624:12345-0@lo:38:kworker.0 Request processed in 30us (91us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318973.381079:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097 00000100:00100000:3.0:1767318973.381082:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 299 00000100:00100000:3.0:1767318973.381086:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982698496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318973.381095:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318973.381119:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982698496:12345-0@lo:38:kworker.0 Request processed in 32us (182us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318973.381125:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 299 00000100:00080000:3.0:1767318973.381156:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318973.381160:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318973.381162:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982698496:0@lo:38:kworker.0 00000100:00080000:3.0:1767318973.381169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318973.381172:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318973.381174:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982698624:0@lo:38:kworker.0 00010000:00080000:0.0:1767318973.381387:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988380 x1853163982697984/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318989 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318973.381399:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982697984:0@lo:400:kworker.0 00000100:00100000:1.0:1767318973.381485:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 837 00000100:00100000:1.0:1767318973.381492:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982698240:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318973.381539:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982698240:12345-0@lo:400:kworker.0 Request processed in 48us (802us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318973.381545:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 837 00010000:00080000:3.0:1767318973.381609:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198b800 x1853163982698240/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318989 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767318973.381620:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982698240:0@lo:400:kworker.0 00000100:00080000:2.0:1767318973.382810:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318973.575312:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318974.596915:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318975.375903:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318975.376019:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318975.376396:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598976 00000100:00100000:2.0:1767318975.376410:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598976 00000100:00100000:2.0:1767318975.376510:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915598848 00000100:00100000:2.0:1767318975.376514:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915598848 00000100:00100000:2.0:1767318975.376537:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 838 00000100:00100000:2.0:1767318975.376549:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915598976:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767318975.376862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318975.376909:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1098 00000100:00100000:1.0:1767318975.376917:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915598848:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318975.376943:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318975.376981:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915598848:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (961us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318975.376989:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1098 00000100:00100000:2.0:1767318975.378166:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915598976:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1608us (2268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318975.378181:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 838 00010000:00010000:1.0:1767318975.622825:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318976.644624:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318977.669585:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318977.988655:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318977.988698:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982698752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318977.988803:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318977.988969:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698752 00000100:00100000:0.0:1767318977.988978:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698752 00000100:00100000:0.0:1767318977.989064:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 578 00000100:00100000:0.0:1767318977.989076:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982698752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318977.989116:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318977.989175:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982698752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318977.989182:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 578 00000100:00080000:2.0:1767318977.989250:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767318977.989365:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982698752:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318977.993741:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982698880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318977.993850:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318977.994103:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982698880 00000100:00100000:2.0:1767318977.994114:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982698880 00000100:00100000:2.0:1767318977.994197:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 579 00000100:00100000:2.0:1767318977.994208:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982698880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767318977.994259:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318977.994353:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982698880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 146us (502us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318977.994361:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 579 00000100:00080000:1.0:1767318977.994702:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767318977.994757:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982698880:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767318978.505515:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318978.505530:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318978.505619:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318978.505624:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318978.505631:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318978.505641:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4221 00000100:00080000:0.0:1767318978.505647:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318978.505656:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318978.505684:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1880 x1853163982699136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318978.505939:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318978.505960:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982699008:0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.506049:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318978.506154:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699008 00000100:00100000:1.0:1767318978.506164:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699008 00000100:00100000:1.0:1767318978.506237:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839 00000100:00100000:1.0:1767318978.506248:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982699008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318978.506360:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982699008:12345-0@lo:400:kworker.0 Request processed in 103us (298us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318978.506370:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839 00000100:00080000:1.0:1767318978.506398:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318978.506434:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435bb80 x1853163982699008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318994 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318978.506448:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982699008:0@lo:400:kworker.0 00000100:00080000:0.0:1767318978.507026:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318978.507034:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318978.507036:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318978.507041:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4221 00000100:00080000:0.0:1767318978.507044:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318978.507050:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318978.507079:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163982699392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318978.507092:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318978.507095:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318978.507096:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318978.507100:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4221 00000100:00080000:0.0:1767318978.507102:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318978.507106:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318978.507120:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50000 x1853163982699520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318978.507157:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318978.507162:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318978.507165:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318978.507169:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4221 00000100:00080000:0.0:1767318978.507171:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318978.507175:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318978.507187:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015180 x1853163982699904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318978.507223:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982699648:0@lo:400:kworker.0 00000100:00100000:0.0:1767318978.507297:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.507339:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318978.507437:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982699264:0@lo:400:kworker.0 00000100:00100000:0.0:1767318978.507468:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318978.507503:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699648 00000100:00100000:0.0:1767318978.507509:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699648 00000100:00100000:0.0:1767318978.507548:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 543 00000100:00100000:0.0:1767318978.507556:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982699648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318978.507600:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982699648:12345-0@lo:400:kworker.0 Request processed in 44us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318978.507607:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 543 00000100:00100000:0.0:1767318978.507647:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982699648:0@lo:400:kworker.0 00000100:00100000:0.0:1767318978.507666:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699264 00000100:00100000:0.0:1767318978.507669:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699264 00000100:00100000:0.0:1767318978.508506:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 840 00000100:00100000:0.0:1767318978.508521:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982699264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318978.508591:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982699264:12345-0@lo:400:kworker.0 Request processed in 76us (1122us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318978.508597:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 840 00000100:00080000:0.0:1767318978.508626:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767318978.508652:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1f80 x1853163982699264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318994 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318978.508664:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982699264:0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.508947:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767318978.508964:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982699776:0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.509008:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318978.509050:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699776 00000100:00100000:2.0:1767318978.509056:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699776 00000100:00100000:2.0:1767318978.509090:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 544 00000100:00100000:2.0:1767318978.509097:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982699776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318978.509138:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982699776:12345-0@lo:400:kworker.0 Request processed in 41us (129us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318978.509144:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 544 00000100:00100000:2.0:1767318978.509172:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982699776:0@lo:400:kworker.0 00000100:00100000:3.0:1767318978.510539:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982699136:0@lo:38:kworker.0 00000100:00100000:3.0:1767318978.510612:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318978.510754:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982699392:0@lo:38:kworker.0 00000100:00100000:3.0:1767318978.510778:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318978.516197:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982699520:0@lo:38:kworker.0 00000100:00100000:3.0:1767318978.516309:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318978.516345:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982699904:0@lo:38:kworker.0 00000100:00100000:3.0:1767318978.516385:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318978.516443:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699520 00000100:00100000:1.0:1767318978.516455:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699520 00000100:00100000:1.0:1767318978.516509:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 300 00000100:00100000:1.0:1767318978.516518:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982699520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318978.516545:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318978.516597:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982699520:12345-0@lo:38:kworker.0 Request processed in 78us (287us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318978.516604:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 300 00000100:00100000:2.0:1767318978.518527:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699136 00000100:00100000:2.0:1767318978.518546:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699136 00000100:00100000:2.0:1767318978.519772:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699392 00000100:00100000:2.0:1767318978.519790:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699392 00000100:00100000:0.0:1767318978.524947:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982699904 00000100:00100000:0.0:1767318978.525001:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982699904 00000100:00100000:0.0:1767318978.525104:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099 00000100:00100000:0.0:1767318978.525115:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699136:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.525151:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.525267:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699136:12345-0@lo:38:kworker.0 Request processed in 143us (14644us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.525288:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099 00000100:00100000:0.0:1767318978.525310:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100 00000100:00100000:0.0:1767318978.525580:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.525593:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.525631:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699392:12345-0@lo:38:kworker.0 Request processed in 54us (14852us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.525637:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100 00000100:00100000:0.0:1767318978.525656:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101 00000100:00100000:0.0:1767318978.525664:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318978.525672:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318978.525743:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982699904:12345-0@lo:38:kworker.0 Request processed in 37us (9312us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318978.525750:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101 00000100:00080000:0.0:1767318978.525846:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318978.525856:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318978.525861:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982699904:0@lo:38:kworker.0 00000100:00080000:0.0:1767318978.525901:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318978.525905:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318978.525908:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982699136:0@lo:38:kworker.0 00000100:00080000:0.0:1767318978.525917:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318978.525921:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318978.525927:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982699392:0@lo:38:kworker.0 00000100:00080000:0.0:1767318978.525939:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318978.525992:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318978.526020:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982699520:0@lo:38:kworker.0 00010000:00010000:1.0:1767318978.696379:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318979.718194:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318980.493811:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318980.493870:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318980.498006:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599104 00000100:00100000:1.0:1767318980.498702:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599104 00000100:00100000:1.0:1767318980.498771:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599232 00000100:00100000:1.0:1767318980.498776:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599232 00000100:00100000:1.0:1767318980.498847:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 841 00000100:00100000:1.0:1767318980.498860:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915599232:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767318980.498969:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915599232:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 111us (5097us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318980.498978:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 841 00000100:00100000:1.0:1767318980.499001:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1102 00000100:00100000:1.0:1767318980.499006:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915599104:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767318980.499031:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318980.499073:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915599104:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 65us (5280us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318980.499319:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1102 00000100:00080000:0.0:1767318980.499799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318980.741158:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318981.765964:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318982.788742:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318983.108567:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982700032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318983.108673:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318983.108690:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767318983.108709:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982700160:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318983.108750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318983.108799:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700032 00000100:00100000:3.0:1767318983.108805:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700160 00000100:00100000:0.0:1767318983.108808:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700032 00000100:00100000:3.0:1767318983.108813:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700160 00000100:00100000:0.0:1767318983.108959:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 580 00000100:00100000:0.0:1767318983.108971:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982700032:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767318983.108994:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 581 00000100:00100000:3.0:1767318983.109002:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982700160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318983.109016:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:3.0:1767318983.109022:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767318983.109072:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982700160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (321us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318983.109081:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 581 00000100:00100000:0.0:1767318983.109082:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982700032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 111us (406us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318983.109092:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 580 00000100:00100000:3.0:1767318983.109134:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982700160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767318983.109189:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982700032:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767318983.109322:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318983.109329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767318983.620761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318983.620778:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767318983.620864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318983.620869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318983.620874:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318983.620885:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4227 00000100:00080000:0.0:1767318983.620891:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318983.620899:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318983.620952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163982700416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318983.620991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318983.620994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318983.620996:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318983.621000:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4227 00000100:00080000:0.0:1767318983.621013:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318983.621018:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318983.621031:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51180 x1853163982700672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767318983.621041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318983.621045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318983.621047:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767318983.621049:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767318983.621052:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4227 00000100:00080000:0.0:1767318983.621057:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318983.621061:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318983.621077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982700800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318983.621078:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982700288:0@lo:400:kworker.0 00000100:00080000:0.0:1767318983.621115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767318983.621118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767318983.621119:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767318983.621123:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4227 00000100:00080000:0.0:1767318983.621125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767318983.621128:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767318983.621141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4a80 x1853163982701184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318983.621190:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982700416:0@lo:38:kworker.0 00000100:00100000:2.0:1767318983.621198:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.621309:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318983.621323:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982701056:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621363:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.621442:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982700544:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621468:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.621498:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318983.621503:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982700928:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621526:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.621570:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700288 00000100:00100000:1.0:1767318983.621580:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700288 00000100:00100000:1.0:1767318983.621646:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700544 00000100:00100000:1.0:1767318983.621650:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700544 00000100:00100000:2.0:1767318983.621663:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842 00000100:00100000:2.0:1767318983.621679:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982700288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621685:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 843 00000100:00100000:1.0:1767318983.621695:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982700544:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621752:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982700544:12345-0@lo:400:kworker.0 Request processed in 58us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318983.621758:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982700288:12345-0@lo:400:kworker.0 Request processed in 82us (562us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318983.621765:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 843 00000100:00080000:1.0:1767318983.621796:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318983.621834:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f50000 x1853163982700544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318999 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318983.621855:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982700544:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.621914:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701056 00000100:00100000:1.0:1767318983.621919:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701056 00010000:00080000:3.0:1767318983.621928:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1f80 x1853163982700288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767318999 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318983.621946:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700928 00000100:00100000:1.0:1767318983.621950:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700928 00000100:00100000:1.0:1767318983.621971:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 545 00000100:00100000:1.0:1767318983.621977:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982701056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.622017:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982701056:12345-0@lo:400:kworker.0 Request processed in 39us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318983.622023:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 545 00000100:00100000:1.0:1767318983.622035:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 546 00000100:00100000:1.0:1767318983.622041:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982700928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.622074:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982700928:12345-0@lo:400:kworker.0 Request processed in 34us (548us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318983.622080:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 546 00000100:00100000:1.0:1767318983.622103:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.622129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982700672:0@lo:38:kworker.0 00000100:00100000:1.0:1767318983.622156:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.622184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982700800:0@lo:38:kworker.0 00000100:00100000:1.0:1767318983.622213:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.622237:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982701184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318983.622261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318983.622317:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700800 00000100:00100000:1.0:1767318983.622321:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700800 00000100:00100000:1.0:1767318983.622341:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 301 00000100:00100000:1.0:1767318983.622347:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982700800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318983.622369:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.622383:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982700288:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.622412:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982700800:12345-0@lo:38:kworker.0 Request processed in 63us (196us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318983.622419:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 301 00000100:00100000:3.0:1767318983.622427:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700416 00000100:00100000:3.0:1767318983.622432:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700416 00000100:00100000:1.0:1767318983.622446:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982700672 00000100:00080000:0.0:1767318983.622852:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318983.623200:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318983.623206:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982700800:0@lo:38:kworker.0 00000100:00100000:2.0:1767318983.623776:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842 00000100:00100000:2.0:1767318983.623816:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701184 00000100:00100000:2.0:1767318983.623824:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701184 00000100:00100000:2.0:1767318983.623870:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103 00000100:00100000:2.0:1767318983.623878:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982701184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767318983.623901:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318983.623978:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982700928:0@lo:400:kworker.0 00000100:00100000:0.0:1767318983.624921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982701056:0@lo:400:kworker.0 00000100:00100000:1.0:1767318983.625728:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982700672 00000100:00100000:1.0:1767318983.625793:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104 00000100:00100000:1.0:1767318983.625803:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982700672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318983.625829:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318983.627459:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982701184:12345-0@lo:38:kworker.0 Request processed in 3569us (5185us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318983.627473:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982700672:12345-0@lo:38:kworker.0 Request processed in 1667us (5311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318983.627482:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103 00000100:00080000:0.0:1767318983.627857:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318983.627865:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318983.627870:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982700672:0@lo:38:kworker.0 00000100:00080000:0.0:1767318983.627886:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318983.627890:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318983.627893:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982701184:0@lo:38:kworker.0 00000100:00100000:1.0:1767318983.628058:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104 00000100:00100000:3.0:1767318983.632122:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1105 00000100:00100000:3.0:1767318983.632139:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982700416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767318983.632204:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767318983.632359:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982700416:12345-0@lo:38:kworker.0 Request processed in 221us (10254us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767318983.632369:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1105 00000100:00080000:0.0:1767318983.633219:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318983.633229:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318983.633235:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982700416:0@lo:38:kworker.0 00010000:00010000:1.0:1767318983.812692:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318984.837526:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318985.612655:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318985.612701:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767318985.612721:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915599616 offset 0 length 520: 4 00000100:00100000:3.0:1767318985.612757:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599488 00000100:00100000:3.0:1767318985.612769:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599488 00000100:00100000:2.0:1767318985.612796:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599360 00000100:00100000:2.0:1767318985.612800:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599360 00000100:00080000:1.0:1767318985.612858:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318985.612980:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 844 00000100:00100000:2.0:1767318985.612993:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915599488:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318985.613111:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915599488:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 121us (467us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318985.613119:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 844 00000100:00100000:2.0:1767318985.613150:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1106 00000100:00100000:2.0:1767318985.613154:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915599360:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318985.613175:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318985.613199:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915599360:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (498us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318985.613205:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1106 00010000:00010000:1.0:1767318985.865160:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318986.884555:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318987.912838:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318988.228516:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318988.229206:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982701312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318988.229326:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318988.229664:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982701440:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318988.229705:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318988.230087:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701312 00000100:00100000:3.0:1767318988.230096:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701312 00000100:00080000:1.0:1767318988.230181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318988.234332:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701440 00000100:00100000:0.0:1767318988.234346:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701440 00000100:00100000:0.0:1767318988.234433:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 582 00000100:00100000:0.0:1767318988.234444:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982701312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318988.234479:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318988.234556:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982701312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (5230us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318988.234564:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 582 00000100:00100000:0.0:1767318988.234581:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 583 00000100:00100000:0.0:1767318988.234587:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982701440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318988.234603:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318988.234640:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982701440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (4943us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318988.234645:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 583 00000100:00100000:0.0:1767318988.234711:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982701440:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318988.234851:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767318988.235011:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982701312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318988.742996:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318963/real 1767318963] req@ffff93463a8e0380 x1853163982695808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318988.743026:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e0380 x1853163982695808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767318988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767318988.743047:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318988.743055:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767318988.743059:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982695808:0@lo:250:kworker.0 00000100:00080000:2.0:1767318988.743254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318988.743259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767318988.743265:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318988.743290:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4211 00000100:00080000:2.0:1767318988.743294:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318988.743301:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318988.743334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982701568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318988.743394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318988.743398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318988.743399:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318988.743403:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4232 00000100:00080000:2.0:1767318988.743406:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318988.743410:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318988.743423:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982701824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318988.743463:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318988.743465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318988.743466:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318988.743469:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4232 00000100:00080000:2.0:1767318988.743471:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318988.743473:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318988.743480:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163982702080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318988.743485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318988.743488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318988.743489:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318988.743491:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4232 00000100:00080000:2.0:1767318988.743493:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318988.743496:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318988.743506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f800 x1853163982702208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767318988.743524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767318988.743527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767318988.743529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767318988.743532:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4232 00000100:00080000:2.0:1767318988.743535:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767318988.743538:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767318988.743547:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ce00 x1853163982702592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767318988.743570:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982701952:0@lo:400:kworker.0 00000100:00100000:2.0:1767318988.743619:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318988.743661:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318988.743665:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982702336:0@lo:400:kworker.0 00000100:00100000:2.0:1767318988.743680:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318988.743701:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701952 00000100:00100000:2.0:1767318988.743706:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701952 00000100:00080000:1.0:1767318988.743812:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767318988.743827:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982702464:0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.743847:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318988.743861:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318988.743864:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982701696:0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.743882:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318988.743905:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702336 00000100:00100000:1.0:1767318988.743907:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702336 00000100:00100000:1.0:1767318988.743945:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702464 00000100:00100000:1.0:1767318988.743947:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702464 00000100:00100000:1.0:1767318988.743959:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 547 00000100:00100000:1.0:1767318988.743963:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982702336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.743998:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982702336:12345-0@lo:400:kworker.0 Request processed in 35us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318988.744002:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 547 00000100:00100000:1.0:1767318988.744015:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 548 00000100:00100000:1.0:1767318988.744020:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982702464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.744046:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982702464:12345-0@lo:400:kworker.0 Request processed in 26us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318988.744051:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 548 00000100:00100000:1.0:1767318988.744079:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701696 00000100:00100000:1.0:1767318988.744083:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701696 00000100:00100000:1.0:1767318988.744110:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 845 00000100:00100000:1.0:1767318988.744114:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982701696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.744134:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982701696:12345-0@lo:400:kworker.0 Request processed in 20us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318988.744138:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 845 00000100:00080000:1.0:1767318988.744151:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767318988.744166:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015180 x1853163982701696/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319004 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318988.744174:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982701696:0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.744180:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982702464:0@lo:400:kworker.0 00000100:00100000:1.0:1767318988.744210:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982702336:0@lo:400:kworker.0 00000100:00100000:2.0:1767318988.744461:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 846 00000100:00100000:2.0:1767318988.744466:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982701952:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767318988.744498:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982701952:12345-0@lo:400:kworker.0 Request processed in 32us (878us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318988.744503:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 846 00000100:00100000:0.0:1767318988.744833:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982701568:0@lo:250:kworker.0 00000400:00000100:0.0:1767318988.744852:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982701568 offset 0 length 520: 4 00000100:00100000:0.0:1767318988.744866:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982701824:0@lo:38:kworker.0 00000100:00100000:0.0:1767318988.744879:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318988.744895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982702080:0@lo:38:kworker.0 00000100:00100000:0.0:1767318988.744906:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318988.744918:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982702208:0@lo:38:kworker.0 00000100:00100000:0.0:1767318988.744939:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318988.744951:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982702592:0@lo:38:kworker.0 00000100:00100000:0.0:1767318988.744962:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318988.744980:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702208 00000100:00100000:0.0:1767318988.744983:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702208 00000100:00100000:0.0:1767318988.744997:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 302 00000100:00100000:0.0:1767318988.745001:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982702208:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318988.745016:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318988.745046:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982702208:12345-0@lo:38:kworker.0 Request processed in 43us (105us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318988.745052:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 302 00000100:00100000:0.0:1767318988.745083:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982701824 00000100:00100000:0.0:1767318988.745086:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982701824 00000100:00100000:0.0:1767318988.745102:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702080 00000100:00100000:0.0:1767318988.745104:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702080 00000100:00100000:0.0:1767318988.745111:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702592 00000100:00100000:0.0:1767318988.745113:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702592 00000100:00100000:0.0:1767318988.745119:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107 00000100:00100000:0.0:1767318988.745122:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982701824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318988.745128:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318988.745144:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982701824:12345-0@lo:38:kworker.0 Request processed in 22us (265us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318988.745148:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107 00000100:00100000:0.0:1767318988.745154:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108 00000100:00100000:0.0:1767318988.745156:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982702080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318988.745170:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318988.745183:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982702080:12345-0@lo:38:kworker.0 Request processed in 27us (277us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318988.745187:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108 00000100:00100000:0.0:1767318988.745192:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109 00000100:00100000:0.0:1767318988.745195:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982702592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318988.745199:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318988.745209:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982702592:12345-0@lo:38:kworker.0 Request processed in 14us (247us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318988.745212:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109 00000100:00080000:0.0:1767318988.745237:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318988.745434:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318988.745437:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982701824:0@lo:38:kworker.0 00000100:00080000:0.0:1767318988.745447:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318988.745449:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318988.745450:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982702080:0@lo:38:kworker.0 00000100:00080000:0.0:1767318988.745455:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318988.745457:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318988.745458:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982702208:0@lo:38:kworker.0 00000100:00080000:0.0:1767318988.745462:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318988.745465:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318988.745466:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982702592:0@lo:38:kworker.0 00010000:00080000:1.0:1767318988.745575:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163982701952/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319004 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318988.745583:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982701952:0@lo:400:kworker.0 00010000:00010000:1.0:1767318988.932563:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318989.957300:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767318990.738228:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767318990.738632:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767318990.739585:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599872 00000100:00100000:2.0:1767318990.739597:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599872 00000100:00100000:2.0:1767318990.739718:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915599744 00000100:00100000:2.0:1767318990.739724:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915599744 00000100:00100000:2.0:1767318990.739748:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 847 00000100:00100000:2.0:1767318990.739757:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915599872:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767318990.739846:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915599872:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (1629us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318990.739854:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 847 00000100:00100000:2.0:1767318990.739872:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1110 00000100:00100000:2.0:1767318990.739876:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915599744:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767318990.739896:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767318990.739920:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915599744:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (1289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767318990.739926:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1110 00000100:00080000:1.0:1767318990.742392:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318990.984361:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318992.007044:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318993.029837:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767318993.348678:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982702720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318993.348759:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318993.348851:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702720 00000100:00100000:0.0:1767318993.348860:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702720 00000100:00100000:0.0:1767318993.348918:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 584 00000100:00100000:0.0:1767318993.348926:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982702720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318993.348955:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318993.349018:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982702720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318993.349025:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 584 00000100:00100000:0.0:1767318993.349064:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982702720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767318993.349580:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982702848:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767318993.349610:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318993.349674:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702848 00000100:00100000:2.0:1767318993.349679:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702848 00000100:00100000:2.0:1767318993.349712:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 585 00000100:00100000:2.0:1767318993.349720:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982702848:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318993.349721:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318993.349728:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767318993.349739:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767318993.349780:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982702848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (169us total) trans 0 rc 0/0 00000100:00100000:2.0:1767318993.349787:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 585 00000100:00100000:2.0:1767318993.349817:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982702848:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767318993.860553:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318993.860561:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318993.860625:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767318993.860889:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318993.860893:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318993.860896:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318993.860902:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4237 00000100:00080000:3.0:1767318993.860905:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318993.860910:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318993.860930:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163982703104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318993.860987:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318993.860991:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318993.860993:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318993.860997:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4237 00000100:00080000:3.0:1767318993.860999:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318993.861004:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318993.861015:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982703360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767318993.861023:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318993.861026:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318993.861027:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318993.861031:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4237 00000100:00080000:3.0:1767318993.861033:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318993.861036:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318993.861046:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581880 x1853163982703488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318993.861139:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982703232:0@lo:400:kworker.0 00000100:00100000:1.0:1767318993.861204:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318993.861237:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767318993.861242:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982703616:0@lo:400:kworker.0 00000100:00100000:1.0:1767318993.861265:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318993.861319:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703616 00000100:00100000:1.0:1767318993.861327:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703616 00000100:00100000:1.0:1767318993.861370:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 549 00000100:00100000:1.0:1767318993.861379:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982703616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318993.861426:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982703616:12345-0@lo:400:kworker.0 Request processed in 47us (159us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318993.861433:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 549 00000100:00100000:1.0:1767318993.861476:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982703616:0@lo:400:kworker.0 00000100:00100000:0.0:1767318993.863034:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703232 00000100:00100000:0.0:1767318993.863050:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703232 00000100:00100000:0.0:1767318993.863128:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 848 00000100:00100000:0.0:1767318993.863140:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982703232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318993.863219:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982703232:12345-0@lo:400:kworker.0 Request processed in 84us (2015us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318993.863226:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 848 00000100:00100000:0.0:1767318993.863260:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982703104:0@lo:38:kworker.0 00000100:00100000:0.0:1767318993.863322:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318993.863345:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982703360:0@lo:38:kworker.0 00000100:00100000:0.0:1767318993.863367:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318993.863383:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982703488:0@lo:38:kworker.0 00000100:00100000:0.0:1767318993.863407:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318993.863437:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703488 00000100:00100000:0.0:1767318993.863442:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703488 00000100:00100000:0.0:1767318993.863461:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 303 00000100:00100000:0.0:1767318993.863467:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982703488:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318993.863484:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318993.863515:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982703488:12345-0@lo:38:kworker.0 Request processed in 47us (106us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318993.863521:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 303 00000100:00100000:0.0:1767318993.863539:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703104 00000100:00100000:0.0:1767318993.863543:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703104 00000100:00100000:0.0:1767318993.863561:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703360 00000100:00100000:0.0:1767318993.863564:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703360 00000100:00100000:0.0:1767318993.863579:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1111 00000100:00100000:0.0:1767318993.863583:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982703104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318993.863591:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318993.863616:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982703104:12345-0@lo:38:kworker.0 Request processed in 32us (294us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318993.863622:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1111 00000100:00100000:0.0:1767318993.863633:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112 00000100:00100000:0.0:1767318993.863637:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982703360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318993.863645:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318993.863667:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982703360:12345-0@lo:38:kworker.0 Request processed in 28us (299us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318993.863672:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112 00000100:00080000:0.0:1767318993.863701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318993.863706:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318993.863709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982703104:0@lo:38:kworker.0 00000100:00080000:0.0:1767318993.863717:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318993.863720:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318993.863723:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982703360:0@lo:38:kworker.0 00000100:00080000:0.0:1767318993.863730:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318993.863733:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318993.863736:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982703488:0@lo:38:kworker.0 00010000:00080000:0.0:1767318993.863772:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580a80 x1853163982703232/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319009 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767318993.863785:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982703232:0@lo:400:kworker.0 00000100:00080000:1.0:1767318993.863879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767318993.864727:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982703744:0@lo:400:kworker.0 00000100:00100000:2.0:1767318993.864787:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767318993.864822:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982702976:0@lo:400:kworker.0 00000100:00100000:2.0:1767318993.864846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318993.864939:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703744 00000100:00100000:1.0:1767318993.864945:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703744 00000100:00100000:1.0:1767318993.864999:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 550 00000100:00100000:1.0:1767318993.865010:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982703744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318993.865060:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982703744:12345-0@lo:400:kworker.0 Request processed in 51us (273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318993.865067:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 550 00000100:00100000:2.0:1767318993.865146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982703744:0@lo:400:kworker.0 00000100:00100000:0.0:1767318993.865216:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982702976 00000100:00100000:0.0:1767318993.865220:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982702976 00000100:00100000:0.0:1767318993.865266:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 849 00000100:00100000:0.0:1767318993.865292:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982702976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767318993.865329:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982702976:12345-0@lo:400:kworker.0 Request processed in 57us (482us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318993.865334:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 849 00010000:00080000:2.0:1767318993.865401:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2680 x1853163982702976/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319009 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318993.865411:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982702976:0@lo:400:kworker.0 00000100:00080000:1.0:1767318993.865464:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767318993.865669:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767318993.865674:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767318993.865678:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767318993.865683:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4237 00000100:00080000:3.0:1767318993.865686:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767318993.865690:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767318993.865712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374e00 x1853163982703872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767318993.865808:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982703872:0@lo:38:kworker.0 00000100:00100000:0.0:1767318993.865835:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318993.865900:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982703872 00000100:00100000:1.0:1767318993.865903:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982703872 00000100:00100000:1.0:1767318993.865924:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113 00000100:00100000:1.0:1767318993.865929:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982703872:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318993.865946:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318993.865994:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982703872:12345-0@lo:38:kworker.0 Request processed in 64us (159us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318993.865999:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113 00000100:00080000:3.0:1767318993.866088:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767318993.866092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767318993.866094:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982703872:0@lo:38:kworker.0 00010000:00010000:1.0:1767318994.053734:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318995.076501:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767318995.851488:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767318995.851577:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767318995.851758:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600128 00000100:00100000:0.0:1767318995.851768:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600128 00000100:00100000:0.0:1767318995.851832:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600000 00000100:00100000:0.0:1767318995.851836:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600000 00000100:00100000:0.0:1767318995.851857:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 850 00000100:00100000:0.0:1767318995.851867:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915600128:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767318995.851948:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915600128:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (470us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318995.851956:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 850 00000100:00100000:0.0:1767318995.851974:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1114 00000100:00100000:0.0:1767318995.851979:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915600000:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767318995.851998:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318995.852024:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915600000:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (447us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318995.852029:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1114 00000100:00080000:1.0:1767318995.852592:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767318996.100802:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318997.128988:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767318998.148557:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767318998.468636:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767318998.468660:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982704000:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767318998.468731:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318998.468945:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704000 00000100:00100000:0.0:1767318998.468953:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704000 00000100:00100000:0.0:1767318998.469013:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 586 00000100:00100000:0.0:1767318998.469024:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982704000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767318998.469058:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318998.469109:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982704000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (378us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318998.469117:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 586 00000100:00080000:0.0:1767318998.469139:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767318998.469176:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982704000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767318998.469244:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982704128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767318998.469297:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767318998.469681:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704128 00000100:00100000:0.0:1767318998.469688:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704128 00000100:00100000:0.0:1767318998.469739:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 587 00000100:00100000:0.0:1767318998.469747:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982704128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767318998.469768:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767318998.469807:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982704128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (509us total) trans 0 rc 0/0 00000100:00100000:0.0:1767318998.469813:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 587 00000100:00100000:0.0:1767318998.469847:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982704128:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767318998.469955:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767318998.980517:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318998.980529:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767318998.981093:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982704256:0@lo:400:kworker.0 00000100:00100000:2.0:1767318998.981164:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767318998.981246:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704256 00000100:00100000:3.0:1767318998.981254:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704256 00000100:00100000:3.0:1767318998.981347:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 851 00000100:00100000:3.0:1767318998.981356:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982704256:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767318998.981440:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318998.981445:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318998.981450:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318998.981455:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4242 00000100:00080000:1.0:1767318998.981458:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318998.981463:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318998.981488:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf100 x1853163982704384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318998.981543:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318998.981547:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318998.981549:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318998.981552:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4242 00000100:00080000:1.0:1767318998.981555:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318998.981558:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318998.981580:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc380 x1853163982704640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318998.981589:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318998.981592:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318998.981594:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318998.981598:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4242 00000100:00080000:1.0:1767318998.981600:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318998.981604:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318998.981614:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc000 x1853163982704768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767318998.981624:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767318998.981626:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767318998.981628:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767318998.981631:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4242 00000100:00080000:1.0:1767318998.981633:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767318998.981635:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767318998.981645:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf800 x1853163982704896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767318998.981681:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982704384:0@lo:38:kworker.0 00000100:00100000:1.0:1767318998.981709:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318998.981737:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982704640:0@lo:38:kworker.0 00000100:00100000:1.0:1767318998.981759:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318998.981773:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982704768:0@lo:38:kworker.0 00000100:00100000:1.0:1767318998.981794:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318998.981908:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982704896:0@lo:38:kworker.0 00000100:00100000:1.0:1767318998.981933:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318998.981964:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767318998.981971:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982704512:0@lo:400:kworker.0 00000100:00100000:1.0:1767318998.981991:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767318998.982024:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704384 00000100:00100000:1.0:1767318998.982029:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704384 00000100:00100000:1.0:1767318998.982045:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704640 00000100:00100000:1.0:1767318998.982047:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704640 00000100:00100000:1.0:1767318998.982053:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704896 00000100:00100000:1.0:1767318998.982056:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704896 00000100:00100000:1.0:1767318998.982062:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704512 00000100:00100000:1.0:1767318998.982064:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704512 00000100:00100000:1.0:1767318998.982082:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 852 00000100:00100000:1.0:1767318998.982088:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982704512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767318998.982126:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982704512:12345-0@lo:400:kworker.0 Request processed in 38us (133us total) trans 0 rc 0/0 00000100:00100000:1.0:1767318998.982132:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 852 00000100:00100000:1.0:1767318998.982149:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115 00000100:00100000:1.0:1767318998.982153:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318998.982169:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318998.982196:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704384:12345-0@lo:38:kworker.0 Request processed in 42us (487us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318998.982202:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115 00000100:00100000:1.0:1767318998.982212:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116 00000100:00100000:1.0:1767318998.982216:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318998.982225:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318998.982246:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704640:12345-0@lo:38:kworker.0 Request processed in 29us (486us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318998.982251:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116 00000100:00100000:1.0:1767318998.982261:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1117 00000100:00100000:1.0:1767318998.982265:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767318998.982298:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767318998.982321:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982704896:12345-0@lo:38:kworker.0 Request processed in 55us (387us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767318998.982326:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1117 00010000:00080000:1.0:1767318998.982355:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cdf80 x1853163982704512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319014 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767318998.982368:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982704512:0@lo:400:kworker.0 00000100:00080000:1.0:1767318998.982393:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318998.982397:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318998.982400:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982704384:0@lo:38:kworker.0 00000100:00080000:1.0:1767318998.982407:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318998.982410:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318998.982412:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982704640:0@lo:38:kworker.0 00000100:00080000:1.0:1767318998.982419:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767318998.982422:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767318998.982425:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982704896:0@lo:38:kworker.0 00000100:00100000:0.0:1767318998.982535:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982704768 00000100:00100000:0.0:1767318998.982540:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982704768 00000100:00100000:0.0:1767318998.982572:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 304 00000100:00100000:0.0:1767318998.982577:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982704768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767318998.982587:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767318998.982623:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982704768:12345-0@lo:38:kworker.0 Request processed in 45us (827us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767318998.982629:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 304 00000100:00080000:0.0:1767318998.982648:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767318998.982664:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767318998.982668:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767318998.982670:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982704768:0@lo:38:kworker.0 00000100:00100000:3.0:1767318998.987086:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982704256:12345-0@lo:400:kworker.0 Request processed in 3144us (3335us total) trans 0 rc 0/0 00000100:00100000:3.0:1767318998.987109:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 851 00010000:00080000:2.0:1767318998.988639:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cea00 x1853163982704256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319014 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767318998.988669:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982704256:0@lo:400:kworker.0 00010000:00010000:1.0:1767318999.172743:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319000.197217:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319000.978397:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319000.978525:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600384 00000100:00100000:0.0:1767319000.978539:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600384 00000100:00100000:3.0:1767319000.978551:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:3.0:1767319000.978705:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319000.981338:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600256 00000100:00100000:0.0:1767319000.981360:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600256 00000100:00100000:1.0:1767319000.981488:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 853 00000100:00100000:1.0:1767319000.982192:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915600384:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319000.982604:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915600384:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1100us (4231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319000.984921:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1118 00000100:00100000:0.0:1767319000.984938:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915600256:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319000.984974:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319000.985079:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915600256:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 141us (6526us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319000.985087:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1118 00000100:00100000:1.0:1767319000.986583:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 853 00010000:00010000:2.0:1767319001.222923:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319002.244929:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319003.268632:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319003.589842:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982705024:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319003.589968:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319003.590033:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705024 00000100:00100000:3.0:1767319003.590049:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705024 00000100:00100000:3.0:1767319003.590159:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 588 00000100:00100000:3.0:1767319003.590170:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982705024:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319003.590205:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319003.590337:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319003.590352:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982705024:12345-0@lo:13:osp-pre-0-1.0 Request processed in 182us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319003.590356:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982705152:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319003.590377:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 588 00000100:00100000:2.0:1767319003.590418:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319003.590478:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319003.594606:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982705024:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319003.598561:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705152 00000100:00100000:1.0:1767319003.598579:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705152 00000100:00100000:1.0:1767319003.598751:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 589 00000100:00100000:1.0:1767319003.598761:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982705152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319003.598803:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319003.598886:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982705152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 127us (8470us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319003.598894:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 589 00000100:00100000:2.0:1767319003.599986:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982705152:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319003.608481:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319004.100442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319004.100458:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319004.100536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319004.100541:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319004.100546:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319004.100558:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4247 00000100:00080000:3.0:1767319004.100564:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319004.100574:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319004.100604:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375180 x1853163982705408/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319004.100665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319004.100668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319004.100671:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319004.100675:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4247 00000100:00080000:3.0:1767319004.100678:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319004.100683:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319004.100698:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375880 x1853163982705664/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319004.100708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319004.100712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319004.100714:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319004.100718:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4247 00000100:00080000:3.0:1767319004.100721:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319004.100728:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982705408:0@lo:38:kworker.0 00000100:00080000:3.0:1767319004.100729:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319004.100748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604376300 x1853163982705792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319004.100763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319004.100767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319004.100769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319004.100773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4247 00000100:00080000:3.0:1767319004.100775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319004.100791:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319004.100811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377480 x1853163982705920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319004.100832:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319004.100854:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982705536:0@lo:400:kworker.0 00000100:00100000:1.0:1767319004.100909:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982705664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.100925:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319004.100959:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319004.101002:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982705792:0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.101021:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705408 00000100:00100000:3.0:1767319004.101033:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705408 00000100:00100000:1.0:1767319004.101040:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319004.101074:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982705920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.101079:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705536 00000100:00100000:3.0:1767319004.101085:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705536 00000100:00100000:1.0:1767319004.101108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319004.101134:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705664 00000100:00100000:3.0:1767319004.101140:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705664 00000100:00100000:3.0:1767319004.101157:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705920 00000100:00100000:3.0:1767319004.101160:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705920 00000100:00100000:1.0:1767319004.101167:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 854 00000100:00100000:1.0:1767319004.101182:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982705536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319004.101244:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982705536:12345-0@lo:400:kworker.0 Request processed in 67us (320us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319004.101472:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982705280:0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.101518:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319004.101559:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705280 00000100:00100000:2.0:1767319004.101563:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705280 00000100:00100000:3.0:1767319004.102512:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767319004.102522:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982705408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319004.102550:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319004.102562:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 854 00000100:00100000:1.0:1767319004.102604:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120 00000100:00100000:1.0:1767319004.102613:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982705664:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.102636:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982705408:12345-0@lo:38:kworker.0 Request processed in 112us (1805us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767319004.102637:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319004.102653:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767319004.102676:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121 00000100:00100000:3.0:1767319004.102682:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982705920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319004.102695:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319004.102706:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982705664:12345-0@lo:38:kworker.0 Request processed in 92us (1749us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319004.102715:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120 00000100:00100000:3.0:1767319004.102744:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982705920:12345-0@lo:38:kworker.0 Request processed in 58us (1634us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319004.102753:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121 00000100:00100000:1.0:1767319004.102756:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982705792 00000100:00100000:1.0:1767319004.102765:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982705792 00000100:00080000:3.0:1767319004.102821:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319004.102822:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 305 00000100:00080000:3.0:1767319004.102838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319004.102841:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982705792:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319004.102849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982705920:0@lo:38:kworker.0 00010000:00020000:1.0:1767319004.102860:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319004.102869:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319004.102877:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319004.102881:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982705408:0@lo:38:kworker.0 00000100:00080000:3.0:1767319004.102895:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319004.102901:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319004.102907:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982705792:12345-0@lo:38:kworker.0 Request processed in 65us (1869us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319004.102908:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982705664:0@lo:38:kworker.0 00000100:00100000:1.0:1767319004.102929:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 305 00000100:00080000:3.0:1767319004.102970:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319004.102978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319004.102984:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982705792:0@lo:38:kworker.0 00010000:00080000:1.0:1767319004.103001:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604376d80 x1853163982705536/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319020 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319004.103017:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982705536:0@lo:400:kworker.0 00000100:00080000:1.0:1767319004.103074:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319004.112667:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 855 00000100:00100000:2.0:1767319004.112701:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982705280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319004.112921:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982705280:12345-0@lo:400:kworker.0 Request processed in 238us (11401us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319004.112930:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 855 00010000:00080000:3.0:1767319004.123121:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604377b80 x1853163982705280/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319020 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319004.123144:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982705280:0@lo:400:kworker.0 00010000:00010000:2.0:1767319004.292736:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319005.317083:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319006.093610:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319006.098479:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600640 00000100:00100000:2.0:1767319006.098505:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600640 00000100:00100000:2.0:1767319006.098641:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 856 00000100:00100000:2.0:1767319006.098653:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915600640:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767319006.098676:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319006.098786:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915600640:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 134us (5191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319006.098794:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 856 00000100:00100000:1.0:1767319006.098991:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319006.099193:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600512 00000100:00100000:3.0:1767319006.099200:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600512 00000100:00100000:3.0:1767319006.099241:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1122 00000100:00100000:3.0:1767319006.099248:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915600512:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319006.099299:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319006.099351:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915600512:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 101us (360us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319006.099358:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1122 00010000:00010000:1.0:1767319006.341523:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319007.364924:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319008.388594:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319008.708897:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319008.708932:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982706048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319008.709036:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319008.709402:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982706176:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319008.709442:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319008.709577:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706048 00000100:00100000:2.0:1767319008.709590:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706048 00000100:00100000:2.0:1767319008.709680:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706176 00000100:00100000:2.0:1767319008.709683:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706176 00000100:00100000:2.0:1767319008.709717:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 590 00000100:00100000:2.0:1767319008.709728:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982706048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319008.709768:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319008.709823:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982706048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (786us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319008.709831:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 590 00000100:00100000:2.0:1767319008.709850:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 591 00000100:00100000:2.0:1767319008.709856:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982706176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319008.709874:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319008.709900:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982706176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319008.709905:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 591 00000100:00100000:3.0:1767319008.709999:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982706176:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319008.710112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319008.710118:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319008.710144:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982706048:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319009.221164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319009.221178:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319009.221307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982706304:0@lo:400:kworker.0 00000100:00100000:1.0:1767319009.221420:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319009.221613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319009.221618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319009.221633:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319009.221641:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4252 00000100:00080000:2.0:1767319009.221645:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319009.221653:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319009.221699:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f100 x1853163982706432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319009.221740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319009.221744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319009.221746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319009.221750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4252 00000100:00080000:2.0:1767319009.221753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319009.221757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319009.221769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982706688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319009.221778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319009.221782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319009.221783:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319009.221787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4252 00000100:00080000:2.0:1767319009.221789:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319009.221793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319009.221810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c380 x1853163982706816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319009.221820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319009.221824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319009.221826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319009.221830:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4252 00000100:00080000:2.0:1767319009.221832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319009.221835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319009.221846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ce00 x1853163982706944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319009.221896:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982706560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319009.221931:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319009.221996:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982706432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319009.229295:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319009.229399:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706304 00000100:00100000:0.0:1767319009.229414:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706304 00000100:00100000:0.0:1767319009.229494:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706560 00000100:00100000:0.0:1767319009.229499:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706560 00000100:00100000:0.0:1767319009.229529:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 857 00000100:00100000:0.0:1767319009.229540:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982706304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319009.229680:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982706304:12345-0@lo:400:kworker.0 Request processed in 132us (8250us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319009.229696:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 857 00000100:00100000:0.0:1767319009.229715:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 858 00000100:00100000:0.0:1767319009.229728:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982706560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319009.229763:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982706560:12345-0@lo:400:kworker.0 Request processed in 37us (7832us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319009.229770:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 858 00010000:00080000:0.0:1767319009.229858:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396df80 x1853163982706304/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319025 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319009.229876:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982706304:0@lo:400:kworker.0 00000100:00080000:2.0:1767319009.229976:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319009.230004:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396e300 x1853163982706560/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319025 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319009.230013:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982706560:0@lo:400:kworker.0 00000100:00100000:3.0:1767319009.230210:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982706688:0@lo:38:kworker.0 00000100:00100000:3.0:1767319009.231777:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319009.238457:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982706816:0@lo:38:kworker.0 00000100:00100000:3.0:1767319009.238552:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319009.239047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982706944:0@lo:38:kworker.0 00000100:00100000:3.0:1767319009.239087:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319009.239138:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706432 00000100:00100000:1.0:1767319009.239150:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706432 00000100:00100000:1.0:1767319009.239209:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706688 00000100:00100000:1.0:1767319009.239213:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706688 00000100:00100000:1.0:1767319009.239238:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1123 00000100:00100000:1.0:1767319009.239248:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982706432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319009.239298:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319009.239355:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982706432:12345-0@lo:38:kworker.0 Request processed in 107us (10095us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319009.239363:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1123 00000100:00100000:1.0:1767319009.239383:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124 00000100:00100000:1.0:1767319009.239388:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982706688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319009.239398:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319009.239426:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982706688:12345-0@lo:38:kworker.0 Request processed in 36us (7659us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319009.239432:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124 00000100:00100000:2.0:1767319009.239663:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706816 00000100:00100000:2.0:1767319009.239669:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706816 00000100:00100000:2.0:1767319009.239712:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 306 00000100:00100000:2.0:1767319009.239717:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982706816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319009.239728:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319009.239762:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982706816:12345-0@lo:38:kworker.0 Request processed in 44us (1211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319009.239769:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 306 00000100:00080000:3.0:1767319009.241263:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319009.241301:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319009.241306:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982706432:0@lo:38:kworker.0 00000100:00080000:3.0:1767319009.241324:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319009.241328:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319009.241331:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982706688:0@lo:38:kworker.0 00000100:00080000:3.0:1767319009.241341:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319009.241345:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319009.241348:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982706816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319009.241472:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982706944 00000100:00100000:0.0:1767319009.241479:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982706944 00000100:00100000:0.0:1767319009.241525:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125 00000100:00100000:0.0:1767319009.241533:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982706944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319009.241553:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319009.241783:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982706944:12345-0@lo:38:kworker.0 Request processed in 248us (2693us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319009.241791:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125 00000100:00080000:3.0:1767319009.241863:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319009.241868:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319009.241871:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982706944:0@lo:38:kworker.0 00010000:00010000:1.0:1767319009.418370:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319010.188055:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319010.188244:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601024 00000100:00100000:0.0:1767319010.188258:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601024 00000100:00100000:0.0:1767319010.188477:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 859 00000100:00100000:0.0:1767319010.188489:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915601024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319010.188607:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915601024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 120us (568us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319010.188616:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 859 00000400:00000100:0.0:1767319010.189445:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915600768 offset 0 length 520: 4 00000100:00100000:0.0:1767319010.189514:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319010.189659:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915600896 00000100:00100000:3.0:1767319010.189678:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915600896 00000100:00100000:3.0:1767319010.189731:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1126 00000100:00100000:3.0:1767319010.189739:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915600896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319010.189776:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319010.189841:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915600896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 101us (326us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319010.189849:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1126 00000100:00080000:2.0:1767319010.194932:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319010.436620:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319011.461246:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319012.486048:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319013.316422:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767318988/real 1767318988] req@ffff934631988000 x1853163982701568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319013 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319013.317304:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631988000 x1853163982701568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319013 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319013.317331:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319013.317362:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319013.317367:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982701568:0@lo:250:kworker.0 00000100:00080000:2.0:1767319013.317540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319013.317547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319013.317551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319013.317557:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4237 00000100:00080000:2.0:1767319013.317563:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319013.317571:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319013.317624:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163982707072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319013.317683:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982707072:0@lo:250:kworker.0 00000400:00000100:3.0:1767319013.317742:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982707072 offset 0 length 520: 4 00000100:00080000:2.0:1767319013.317853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319013.317857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319013.317859:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319013.317863:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1767319013.317866:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319013.317870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319013.317885:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163982707328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319013.317897:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982707200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319013.317950:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319013.318055:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707200 00000100:00100000:1.0:1767319013.318066:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707200 00000100:00100000:1.0:1767319013.318154:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 860 00000100:00100000:1.0:1767319013.318167:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982707200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319013.318233:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982707200:12345-0@lo:400:kworker.0 Request processed in 69us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319013.318242:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 860 00000100:00080000:0.0:1767319013.318370:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319013.318401:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c000 x1853163982707200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319013.318416:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982707200:0@lo:400:kworker.0 00000100:00100000:3.0:1767319013.319799:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982707328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319013.319877:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319013.320042:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707328 00000100:00100000:1.0:1767319013.320050:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707328 00000100:00100000:1.0:1767319013.320095:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1767319013.320105:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982707328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319013.320130:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319013.320206:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982707328:12345-0@lo:38:kworker.0 Request processed in 101us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319013.320214:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127 00000100:00080000:0.0:1767319013.320299:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319013.320305:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319013.320310:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982707328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319013.320501:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982707456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319013.320535:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319013.320580:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707456 00000100:00100000:1.0:1767319013.320585:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707456 00000100:00100000:1.0:1767319013.320644:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 861 00000100:00100000:1.0:1767319013.320650:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982707456:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319013.320727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319013.320734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319013.320738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319013.320739:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319013.320744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1767319013.320749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319013.320755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319013.320780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163982707584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319013.321068:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982707584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319013.321104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319013.321128:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707584 00000100:00100000:0.0:1767319013.321133:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707584 00000100:00100000:0.0:1767319013.321161:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128 00000100:00100000:0.0:1767319013.321166:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982707584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319013.321180:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319013.321523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319013.321528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319013.321531:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319013.321535:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1767319013.321539:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319013.321544:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319013.321566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163982707712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319013.321634:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982707584:12345-0@lo:38:kworker.0 Request processed in 466us (528us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319013.321643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128 00000100:00080000:0.0:1767319013.321682:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319013.321686:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319013.321689:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982707584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319013.321709:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982707712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319013.321740:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319013.321794:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707712 00000100:00100000:0.0:1767319013.321798:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707712 00000100:00100000:0.0:1767319013.321822:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 307 00000100:00100000:0.0:1767319013.321828:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982707712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319013.321841:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319013.321873:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982707712:12345-0@lo:38:kworker.0 Request processed in 44us (133us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319013.321879:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 307 00000100:00080000:0.0:1767319013.321903:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319013.321907:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319013.321910:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982707712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319013.322145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982707840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319013.322180:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319013.322384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319013.322388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319013.322390:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319013.322394:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1767319013.322397:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319013.322402:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319013.322419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982708096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319013.322455:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707840 00000100:00100000:2.0:1767319013.322460:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707840 00000100:00100000:2.0:1767319013.322512:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 551 00000100:00100000:2.0:1767319013.322519:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982707840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319013.322803:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982707840:12345-0@lo:400:kworker.0 Request processed in 283us (621us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319013.322811:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 551 00000100:00100000:2.0:1767319013.322840:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982707968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319013.322870:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319013.322903:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982707968 00000100:00100000:2.0:1767319013.322907:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982707968 00000100:00100000:2.0:1767319013.322934:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 552 00000100:00100000:2.0:1767319013.322940:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982707968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319013.322977:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982707968:12345-0@lo:400:kworker.0 Request processed in 37us (106us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319013.322984:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 552 00000100:00080000:2.0:1767319013.323002:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319013.323007:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319013.323029:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982707968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319013.323118:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982707840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319013.323139:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982708096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319013.323169:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319013.323298:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708096 00000100:00100000:2.0:1767319013.323303:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708096 00000100:00100000:2.0:1767319013.323327:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1129 00000100:00100000:2.0:1767319013.323332:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982708096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319013.323360:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319013.323532:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982708096:12345-0@lo:38:kworker.0 Request processed in 198us (362us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319013.323539:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1129 00000100:00080000:0.0:1767319013.323639:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319013.323643:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319013.323647:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982708096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319013.324402:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982707456:12345-0@lo:400:kworker.0 Request processed in 3744us (3859us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319013.324415:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 861 00010000:00080000:3.0:1767319013.326030:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396e680 x1853163982707456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319013.326053:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982707456:0@lo:400:kworker.0 00010000:00010000:2.0:1767319013.509459:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319013.828770:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319013.828998:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982708352:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319013.829077:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319013.829252:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708352 00000100:00100000:0.0:1767319013.829264:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708352 00000100:00100000:0.0:1767319013.829359:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 592 00000100:00100000:0.0:1767319013.829371:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982708352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319013.829411:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319013.829474:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982708352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (386us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319013.829483:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 592 00000100:00100000:3.0:1767319013.829576:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982708352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319013.829976:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982708224:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319013.830016:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319013.830308:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708224 00000100:00100000:3.0:1767319013.830315:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708224 00000100:00100000:3.0:1767319013.830393:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 593 00000100:00100000:3.0:1767319013.830401:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982708224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319013.830423:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319013.830489:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982708224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (472us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319013.830496:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 593 00000100:00100000:1.0:1767319013.830574:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982708224:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319014.532639:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319015.311148:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319015.311329:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319015.313120:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601152 00000100:00100000:3.0:1767319015.313137:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601152 00000100:00100000:3.0:1767319015.313187:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601280 00000100:00100000:3.0:1767319015.313190:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601280 00000100:00100000:0.0:1767319015.313238:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1130 00000100:00100000:0.0:1767319015.313251:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915601152:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:1.0:1767319015.313260:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:0.0:1767319015.313315:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319015.313611:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 862 00000100:00100000:3.0:1767319015.313622:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915601280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319015.313627:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915601152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 373us (2493us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319015.313642:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1130 00000100:00100000:3.0:1767319015.313749:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915601280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 129us (2419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319015.313758:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 862 00010000:00010000:1.0:1767319015.557010:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319016.580855:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319017.604470:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319018.437320:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319018.437343:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319018.437428:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319018.437434:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319018.437438:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319018.437449:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4261 00000100:00080000:0.0:1767319018.437456:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319018.437464:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319018.437479:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982708480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319018.437491:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163982708608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319018.437545:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319018.437549:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319018.437551:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319018.437555:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4261 00000100:00080000:0.0:1767319018.437558:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319018.437562:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319018.437575:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163982708864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319018.437583:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319018.437586:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319018.437588:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319018.437591:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4261 00000100:00100000:1.0:1767319018.437591:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319018.437594:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319018.437598:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319018.437657:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982708736:0@lo:400:kworker.0 00000100:00100000:1.0:1767319018.437688:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319018.437726:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708480 00000100:00100000:1.0:1767319018.437736:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708480 00000100:00100000:1.0:1767319018.437809:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708736 00000100:00100000:1.0:1767319018.437813:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708736 00000100:00100000:2.0:1767319018.437815:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982708608:0@lo:38:kworker.0 00000100:00100000:1.0:1767319018.437840:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 863 00000100:00100000:2.0:1767319018.437848:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319018.437849:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d6450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982708480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.437868:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982708864:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.437888:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319018.437908:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d6450 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982708480:12345-0@lo:400:kworker.0 Request processed in 60us (317us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.437911:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708608 00000100:00100000:2.0:1767319018.437914:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708608 00000100:00100000:1.0:1767319018.437916:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 863 00000100:00100000:0.0:1767319018.437950:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163982708992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319018.438020:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319018.438023:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319018.438025:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319018.438030:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4261 00000100:00080000:0.0:1767319018.438032:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319018.438036:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319018.438047:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d968700 x1853163982709376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319018.442451:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708864 00000100:00100000:2.0:1767319018.442475:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708864 00000100:00100000:2.0:1767319018.442812:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 864 00000100:00100000:2.0:1767319018.442835:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982708736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.443018:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982708736:12345-0@lo:400:kworker.0 Request processed in 193us (5328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.443030:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 864 00000100:00100000:2.0:1767319018.443058:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1131 00000100:00100000:2.0:1767319018.443063:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982708608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319018.443089:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319018.443124:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982708608:12345-0@lo:38:kworker.0 Request processed in 59us (5274us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319018.443131:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1131 00000100:00100000:2.0:1767319018.443151:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1132 00000100:00100000:2.0:1767319018.443155:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982708864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319018.443165:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319018.443191:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982708864:12345-0@lo:38:kworker.0 Request processed in 34us (5302us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319018.443197:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1132 00000100:00100000:2.0:1767319018.443242:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982708992:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.443317:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319018.443370:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982709376:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.443396:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319018.443436:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319018.443442:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319018.443446:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982708608:0@lo:38:kworker.0 00000100:00080000:2.0:1767319018.443458:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319018.443463:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319018.443466:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982708864:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.443528:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982708992 00000100:00100000:2.0:1767319018.443798:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982708992 00000100:00100000:2.0:1767319018.443833:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 308 00000100:00100000:2.0:1767319018.443839:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982708992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319018.443851:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319018.443949:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982708992:12345-0@lo:38:kworker.0 Request processed in 109us (632us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319018.443956:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 308 00000100:00080000:2.0:1767319018.443987:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319018.443992:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319018.443995:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982708992:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.444013:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709376 00000100:00100000:2.0:1767319018.444017:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709376 00000100:00100000:2.0:1767319018.444031:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1133 00000100:00100000:2.0:1767319018.444035:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982709376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319018.444044:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319018.444075:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982709376:12345-0@lo:38:kworker.0 Request processed in 38us (677us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319018.444081:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1133 00000100:00080000:2.0:1767319018.444102:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319018.444107:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319018.444110:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982709376:0@lo:38:kworker.0 00000100:00100000:2.0:1767319018.444155:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982709248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.444189:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319018.444227:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709248 00000100:00100000:2.0:1767319018.444232:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709248 00000100:00100000:2.0:1767319018.444267:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 553 00000100:00100000:2.0:1767319018.444298:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982709248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.444335:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982709248:12345-0@lo:400:kworker.0 Request processed in 38us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.444341:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 553 00000100:00100000:2.0:1767319018.444372:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982709248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.444400:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982709120:0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.444424:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319018.444449:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709120 00000100:00100000:2.0:1767319018.444453:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709120 00000100:00100000:2.0:1767319018.444477:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 554 00000100:00100000:2.0:1767319018.444483:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982709120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319018.444529:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982709120:12345-0@lo:400:kworker.0 Request processed in 46us (103us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319018.444535:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 554 00000100:00100000:2.0:1767319018.444572:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982709120:0@lo:400:kworker.0 00000100:00080000:2.0:1767319018.450187:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319018.450202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319018.450207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319018.450307:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581880 x1853163982708480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319034 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319018.450328:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982708480:0@lo:400:kworker.0 00010000:00080000:1.0:1767319018.458556:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580a80 x1853163982708736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319034 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319018.458581:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982708736:0@lo:400:kworker.0 00010000:00010000:1.0:1767319018.628604:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319018.948832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982709504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319018.948940:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319018.952702:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709504 00000100:00100000:3.0:1767319018.952724:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709504 00000100:00100000:3.0:1767319018.952835:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 594 00000100:00100000:3.0:1767319018.952848:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982709504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319018.952890:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319018.953009:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982709632:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319018.953015:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982709504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 168us (4073us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319018.953025:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 594 00000100:00100000:1.0:1767319018.953068:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319018.953361:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982709504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319018.955630:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709632 00000100:00100000:3.0:1767319018.955646:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709632 00000100:00100000:3.0:1767319018.955848:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 595 00000100:00100000:3.0:1767319018.955860:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982709632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319018.956566:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319018.959652:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982709632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3777us (6567us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319018.959679:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 595 00000100:00100000:1.0:1767319018.961748:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982709632:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319019.652670:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319020.432169:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319020.432339:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319020.432759:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601536 00000100:00100000:1.0:1767319020.432760:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601408 00000100:00100000:0.0:1767319020.432776:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601536 00000100:00100000:1.0:1767319020.432776:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601408 00000100:00100000:1.0:1767319020.432926:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1134 00000100:00100000:1.0:1767319020.432939:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915601408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319020.432973:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319020.433077:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915601408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 138us (737us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319020.433086:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1134 00000100:00080000:2.0:1767319020.434525:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319020.434628:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 865 00000100:00100000:0.0:1767319020.434910:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915601536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319020.435524:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915601536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 884us (3370us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319020.435535:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 865 00010000:00010000:1.0:1767319020.678650:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319021.700602:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319022.726538:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319023.558835:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319023.558848:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319023.558930:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319023.558934:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319023.558940:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319023.558950:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4266 00000100:00080000:0.0:1767319023.558956:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319023.558964:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319023.558989:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d969f80 x1853163982709888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319023.559026:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319023.559029:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319023.559031:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319023.559035:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4266 00000100:00080000:0.0:1767319023.559037:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319023.559041:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319023.559050:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d968000 x1853163982710144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319023.559058:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319023.559062:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319023.559064:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319023.559067:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4266 00000100:00080000:0.0:1767319023.559069:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319023.559072:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319023.559082:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d96b800 x1853163982710272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319023.559100:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319023.559103:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319023.559106:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319023.559109:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4266 00000100:00080000:0.0:1767319023.559111:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319023.559115:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319023.559123:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d968e00 x1853163982710656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319023.559162:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982710016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319023.559225:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982709760:0@lo:400:kworker.0 00000100:00100000:0.0:1767319023.559247:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319023.559341:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982710400:0@lo:400:kworker.0 00000100:00100000:3.0:1767319023.559422:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982710528:0@lo:400:kworker.0 00000100:00100000:1.0:1767319023.559489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319023.559541:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709760 00000100:00100000:1.0:1767319023.559541:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710016 00000100:00100000:0.0:1767319023.559550:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709760 00000100:00100000:1.0:1767319023.559550:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710016 00000100:00100000:1.0:1767319023.559610:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 866 00000100:00100000:1.0:1767319023.564797:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982710016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319023.564860:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 867 00000100:00100000:0.0:1767319023.564875:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982709760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319023.564945:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982710016:12345-0@lo:400:kworker.0 Request processed in 5326us (5699us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319023.564954:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 866 00000100:00100000:0.0:1767319023.564962:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982709760:12345-0@lo:400:kworker.0 Request processed in 89us (5471us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319023.564970:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 867 00000100:00080000:1.0:1767319023.565000:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319023.565035:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463d968380 x1853163982710016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319039 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:1.0:1767319023.565051:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463d96b100 x1853163982709760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319039 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319023.565056:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982710016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319023.565067:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982709760:0@lo:400:kworker.0 00000100:00100000:0.0:1767319023.565250:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982709888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319023.565332:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319023.565357:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982710144:0@lo:38:kworker.0 00000100:00100000:1.0:1767319023.565382:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982709888 00000100:00100000:0.0:1767319023.565383:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319023.565393:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982709888 00000100:00100000:0.0:1767319023.565405:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982710272:0@lo:38:kworker.0 00000100:00100000:1.0:1767319023.565430:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710144 00000100:00100000:1.0:1767319023.565434:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710144 00000100:00100000:1.0:1767319023.565453:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1135 00000100:00100000:0.0:1767319023.565454:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319023.565459:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982709888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319023.565481:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319023.565495:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982710656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319023.565515:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982709888:12345-0@lo:38:kworker.0 Request processed in 56us (184us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319023.565521:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1135 00000100:00100000:0.0:1767319023.565530:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319023.565533:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136 00000100:00100000:1.0:1767319023.565537:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982710144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319023.565542:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319023.565564:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982710144:12345-0@lo:38:kworker.0 Request processed in 26us (182us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319023.565568:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136 00000100:00100000:1.0:1767319023.565575:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710656 00000100:00080000:0.0:1767319023.565578:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319023.565579:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710656 00000100:00100000:1.0:1767319023.565588:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137 00000100:00100000:1.0:1767319023.565592:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982710656:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767319023.565597:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:1.0:1767319023.565599:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319023.565604:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982709888:0@lo:38:kworker.0 00000100:00080000:0.0:1767319023.565621:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319023.565621:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982710656:12345-0@lo:38:kworker.0 Request processed in 27us (93us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319023.565630:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319023.565631:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137 00000100:00100000:0.0:1767319023.565641:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982710144:0@lo:38:kworker.0 00000100:00080000:0.0:1767319023.565655:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319023.565659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319023.565663:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982710656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319023.565669:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710272 00000100:00100000:1.0:1767319023.565675:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710272 00000100:00100000:1.0:1767319023.565711:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 309 00000100:00100000:1.0:1767319023.565718:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982710272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319023.565725:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319023.565759:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982710272:12345-0@lo:38:kworker.0 Request processed in 42us (311us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319023.565763:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 309 00000100:00100000:2.0:1767319023.567963:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319023.568040:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710400 00000100:00100000:1.0:1767319023.568051:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710400 00000100:00100000:1.0:1767319023.568122:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 555 00000100:00100000:1.0:1767319023.568134:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982710400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319023.568204:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982710400:12345-0@lo:400:kworker.0 Request processed in 73us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319023.568211:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 555 00000100:00080000:0.0:1767319023.568335:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319023.568343:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319023.568348:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982710272:0@lo:38:kworker.0 00000100:00080000:2.0:1767319023.568427:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319023.568454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982710400:0@lo:400:kworker.0 00000100:00100000:3.0:1767319023.570564:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319023.571884:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710528 00000100:00100000:2.0:1767319023.571902:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710528 00000100:00100000:2.0:1767319023.572176:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 556 00000100:00100000:2.0:1767319023.572188:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982710528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319023.572330:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982710528:12345-0@lo:400:kworker.0 Request processed in 145us (1777us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319023.572338:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 556 00000100:00100000:0.0:1767319023.572450:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982710528:0@lo:400:kworker.0 00000100:00080000:1.0:1767319023.573942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319023.752767:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319024.068734:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982710784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319024.068836:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.069225:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319024.069237:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982710912:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319024.069301:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319024.069333:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710784 00000100:00100000:2.0:1767319024.069341:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710784 00000100:00100000:2.0:1767319024.069395:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982710912 00000100:00100000:2.0:1767319024.069398:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982710912 00000100:00100000:2.0:1767319024.069426:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 596 00000100:00100000:2.0:1767319024.069436:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982710784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319024.069477:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319024.069525:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982710784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (688us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319024.069535:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 596 00000100:00100000:2.0:1767319024.069555:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 597 00000100:00100000:2.0:1767319024.069561:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982710912:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319024.069866:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982710784:0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319024.069925:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319024.070012:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982710912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 450us (710us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319024.070020:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 597 00000100:00100000:3.0:1767319024.073795:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982710912:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319024.774583:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319025.550636:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319025.551345:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319025.552301:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601792 00000100:00100000:0.0:1767319025.552318:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601792 00000100:00080000:2.0:1767319025.553032:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319025.553096:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601664 00000100:00100000:0.0:1767319025.553102:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601664 00000100:00100000:1.0:1767319025.553345:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 868 00000100:00100000:1.0:1767319025.553364:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915601792:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319025.553491:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915601792:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 135us (3013us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319025.553499:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 868 00000100:00100000:1.0:1767319025.553522:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1138 00000100:00100000:1.0:1767319025.553527:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915601664:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319025.553559:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319025.553593:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915601664:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (2251us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319025.553603:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1138 00010000:00010000:2.0:1767319025.797447:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319026.820524:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319027.844631:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319028.677106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319028.677118:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319028.677211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319028.677216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319028.677221:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319028.677233:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4272 00000100:00080000:3.0:1767319028.677239:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319028.677251:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319028.677313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375500 x1853163982711168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319028.677360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319028.677364:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319028.677366:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319028.677371:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4272 00000100:00080000:3.0:1767319028.677374:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319028.677390:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319028.677403:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375880 x1853163982711424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319028.677413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319028.677416:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319028.677418:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319028.677422:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4272 00000100:00080000:3.0:1767319028.677425:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319028.677429:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319028.677442:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377480 x1853163982711552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319028.677465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319028.677469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319028.677471:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319028.677475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4272 00000100:00080000:3.0:1767319028.677477:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319028.677480:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319028.677491:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377100 x1853163982711936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319028.677525:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982711296:0@lo:400:kworker.0 00000100:00100000:3.0:1767319028.677618:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319028.677740:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319028.677747:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982711680:0@lo:400:kworker.0 00000100:00100000:3.0:1767319028.677788:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319028.677832:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711296 00000100:00100000:3.0:1767319028.677844:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711296 00000100:00100000:3.0:1767319028.677937:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 869 00000100:00100000:1.0:1767319028.677955:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982711808:0@lo:400:kworker.0 00000100:00100000:3.0:1767319028.677967:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982711296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319028.678000:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319028.678075:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982711040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319028.678249:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982711168:0@lo:38:kworker.0 00000100:00100000:3.0:1767319028.678475:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982711296:12345-0@lo:400:kworker.0 Request processed in 511us (854us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319028.678483:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 869 00010000:00080000:3.0:1767319028.678532:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604376300 x1853163982711296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319044 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319028.678547:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982711296:0@lo:400:kworker.0 00000100:00100000:3.0:1767319028.678591:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711680 00000100:00100000:3.0:1767319028.678597:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711680 00000100:00100000:3.0:1767319028.678630:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711808 00000100:00100000:3.0:1767319028.678634:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711808 00000100:00100000:3.0:1767319028.678674:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 557 00000100:00100000:3.0:1767319028.678682:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463934ac50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982711680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319028.678722:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463934ac50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982711680:12345-0@lo:400:kworker.0 Request processed in 42us (934us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319028.678728:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 557 00000100:00100000:1.0:1767319028.679292:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 558 00000100:00100000:1.0:1767319028.679306:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982711808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319028.679375:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982711808:12345-0@lo:400:kworker.0 Request processed in 74us (1376us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319028.679384:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 558 00000100:00100000:1.0:1767319028.679434:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982711808:0@lo:400:kworker.0 00000100:00080000:1.0:1767319028.679488:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319028.679495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319028.679498:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319028.679531:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982711680:0@lo:400:kworker.0 00000100:00100000:0.0:1767319028.684288:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319028.684370:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711040 00000100:00100000:1.0:1767319028.684384:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711040 00000100:00100000:2.0:1767319028.684583:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319028.688399:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711168 00000100:00100000:0.0:1767319028.688426:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711168 00000100:00100000:1.0:1767319028.688437:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 870 00000100:00100000:1.0:1767319028.688471:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982711040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319028.688502:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767319028.688511:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982711168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319028.688538:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319028.688608:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982711168:12345-0@lo:38:kworker.0 Request processed in 95us (4023us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319028.688610:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982711040:12345-0@lo:400:kworker.0 Request processed in 159us (4353us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319028.688625:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1139 00000100:00080000:2.0:1767319028.688714:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319028.688721:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319028.688725:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982711168:0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.688739:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982711424:0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.688785:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319028.688829:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982711552:0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.688874:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319028.688912:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711552 00000100:00100000:2.0:1767319028.688916:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711552 00000100:00100000:0.0:1767319028.689489:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711424 00000100:00100000:0.0:1767319028.689495:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711424 00000100:00100000:2.0:1767319028.689548:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 310 00000100:00100000:2.0:1767319028.689554:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982711552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319028.689570:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319028.689614:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982711552:12345-0@lo:38:kworker.0 Request processed in 58us (739us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319028.689620:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 310 00000100:00080000:2.0:1767319028.689666:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319028.689671:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319028.689674:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982711552:0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.689686:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982711936:0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.689715:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319028.689742:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982711936 00000100:00100000:2.0:1767319028.689746:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982711936 00000100:00080000:3.0:1767319028.692214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319028.692423:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604374e00 x1853163982711040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319044 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319028.692443:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982711040:0@lo:400:kworker.0 00000100:00100000:1.0:1767319028.692577:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 870 00000100:00100000:0.0:1767319028.692831:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140 00000100:00100000:0.0:1767319028.692842:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982711424:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319028.693683:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1141 00000100:00100000:2.0:1767319028.693694:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982711936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319028.693720:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319028.693838:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982711936:12345-0@lo:38:kworker.0 Request processed in 143us (4121us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319028.693845:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1141 00000100:00080000:3.0:1767319028.694244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319028.694253:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319028.694257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982711936:0@lo:38:kworker.0 00010000:00020000:0.0:1767319028.694556:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319028.694628:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982711424:12345-0@lo:38:kworker.0 Request processed in 1786us (5842us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319028.694638:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140 00000100:00080000:3.0:1767319028.694760:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319028.694766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319028.694769:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982711424:0@lo:38:kworker.0 00010000:00010000:2.0:1767319028.868736:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319029.190057:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982712064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319029.190261:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319029.190474:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319029.190491:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982712192:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319029.190552:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319029.190878:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712064 00000100:00100000:2.0:1767319029.190889:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712064 00000100:00100000:2.0:1767319029.190945:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712192 00000100:00100000:2.0:1767319029.190949:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712192 00000100:00100000:2.0:1767319029.190967:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 598 00000100:00100000:2.0:1767319029.190977:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982712064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319029.191016:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319029.191072:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982712064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (817us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319029.191081:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 598 00000100:00100000:2.0:1767319029.191100:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 599 00000100:00100000:2.0:1767319029.191106:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982712192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319029.191125:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319029.191160:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982712192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 54us (608us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319029.191174:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 599 00000100:00100000:3.0:1767319029.191258:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982712064:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319029.191348:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982712192:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319029.893614:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319030.669224:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319030.669341:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319030.669347:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602048 00000100:00100000:2.0:1767319030.669358:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602048 00000100:00100000:2.0:1767319030.669438:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915601920 00000100:00080000:3.0:1767319030.669439:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319030.669442:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915601920 00000100:00100000:0.0:1767319030.669457:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 871 00000100:00100000:0.0:1767319030.669485:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602048:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319030.669532:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1142 00000100:00100000:2.0:1767319030.669542:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915601920:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319030.669566:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319030.669623:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602048:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 150us (405us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319030.669635:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 871 00000100:00100000:2.0:1767319030.669643:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915601920:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 84us (284us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319030.669654:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1142 00010000:00010000:1.0:1767319030.916519:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319031.940741:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319032.965338:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319033.796925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319033.796938:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319033.797293:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982712320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.797365:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319033.797523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319033.797528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319033.797532:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319033.797537:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4277 00000100:00080000:3.0:1767319033.797541:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319033.797548:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319033.797572:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374e00 x1853163982712448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319033.797603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319033.797606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319033.797608:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319033.797611:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4277 00000100:00080000:3.0:1767319033.797614:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319033.797618:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319033.797628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375500 x1853163982712704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319033.797641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319033.797644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319033.797645:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319033.797648:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4277 00000100:00080000:3.0:1767319033.797650:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319033.797653:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319033.797663:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377800 x1853163982712832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319033.797724:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982713088:0@lo:400:kworker.0 00000100:00100000:0.0:1767319033.797761:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319033.797796:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982712448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319033.797820:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319033.798908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319033.798916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319033.798918:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319033.798923:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4277 00000100:00080000:3.0:1767319033.798926:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319033.798933:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319033.798942:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982712960:0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.798982:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319033.799505:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319033.799515:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982712576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.799557:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713088 00000100:00100000:1.0:1767319033.799566:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713088 00000100:00100000:1.0:1767319033.799621:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712960 00000100:00100000:1.0:1767319033.799625:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712960 00000100:00100000:1.0:1767319033.799650:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 559 00000100:00100000:1.0:1767319033.799658:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982713088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.799717:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982713088:12345-0@lo:400:kworker.0 Request processed in 59us (1954us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319033.799738:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 559 00000100:00100000:1.0:1767319033.799757:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 560 00000100:00100000:1.0:1767319033.799764:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982712960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.799797:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982712960:12345-0@lo:400:kworker.0 Request processed in 34us (815us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319033.799803:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 560 00000100:00100000:1.0:1767319033.799863:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982713088:0@lo:400:kworker.0 00000100:00080000:1.0:1767319033.799905:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319033.799910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319033.799943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374700 x1853163982713216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319033.803396:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319033.803430:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982712704:0@lo:38:kworker.0 00000100:00100000:2.0:1767319033.803460:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712320 00000100:00100000:2.0:1767319033.803471:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712320 00000100:00100000:2.0:1767319033.803585:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712448 00000100:00100000:2.0:1767319033.803591:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712448 00000100:00100000:1.0:1767319033.803622:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 872 00000100:00100000:1.0:1767319033.803635:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982712320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319033.803699:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982712320:12345-0@lo:400:kworker.0 Request processed in 69us (6334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319033.803706:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 872 00000100:00100000:1.0:1767319033.803736:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143 00000100:00100000:1.0:1767319033.803741:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982712448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319033.803762:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319033.803795:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982712448:12345-0@lo:38:kworker.0 Request processed in 53us (5974us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319033.803802:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143 00010000:00080000:0.0:1767319033.803877:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604377100 x1853163982712320/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319049 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319033.803896:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982712320:0@lo:400:kworker.0 00000100:00080000:0.0:1767319033.803929:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319033.807008:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712576 00000100:00100000:1.0:1767319033.807025:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712576 00000100:00100000:1.0:1767319033.807113:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 873 00000100:00100000:1.0:1767319033.807136:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982712576:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319033.807218:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319033.807526:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319033.807654:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712704 00000100:00100000:2.0:1767319033.807660:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712704 00000100:00100000:2.0:1767319033.807688:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1144 00000100:00100000:2.0:1767319033.807694:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982712704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319033.807716:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319033.807776:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982712704:12345-0@lo:38:kworker.0 Request processed in 80us (250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319033.807861:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1144 00000100:00080000:0.0:1767319033.809658:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319033.809670:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319033.809676:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982712704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319033.809691:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982712832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319033.809752:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319033.809898:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982713216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319033.809929:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319033.809968:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319033.809972:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319033.809975:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982712448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319033.811481:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982712832 00000100:00100000:2.0:1767319033.811496:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982712832 00000100:00100000:2.0:1767319033.811562:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 311 00000100:00100000:2.0:1767319033.811570:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982712832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319033.811593:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319033.811667:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982712832:12345-0@lo:38:kworker.0 Request processed in 97us (1914us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319033.811674:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 311 00000100:00100000:2.0:1767319033.811700:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713216 00000100:00100000:2.0:1767319033.811705:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713216 00000100:00100000:2.0:1767319033.811723:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145 00000100:00100000:2.0:1767319033.811728:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982713216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319033.811738:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319033.811765:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982713216:12345-0@lo:38:kworker.0 Request processed in 36us (1835us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319033.811771:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145 00000100:00100000:1.0:1767319033.811967:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982712576:12345-0@lo:400:kworker.0 Request processed in 4844us (8577us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319033.811975:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 873 00000100:00080000:0.0:1767319033.812236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319033.812242:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319033.812246:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982713216:0@lo:38:kworker.0 00000100:00080000:0.0:1767319033.812257:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319033.812260:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319033.812263:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982712832:0@lo:38:kworker.0 00010000:00080000:3.0:1767319033.812737:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604377480 x1853163982712576/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319049 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319033.812749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982712576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319033.812762:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982712960:0@lo:400:kworker.0 00010000:00010000:1.0:1767319033.990562:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319034.308979:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319034.309001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982713472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319034.309063:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319034.309686:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713472 00000100:00100000:0.0:1767319034.309695:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713472 00000100:00100000:0.0:1767319034.309756:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 600 00000100:00100000:0.0:1767319034.309765:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982713472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319034.309794:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319034.309845:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982713472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (780us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319034.309852:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 600 00000100:00100000:0.0:1767319034.309895:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982713472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319034.310117:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319034.310130:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982713344:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319034.310824:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319034.311009:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713344 00000100:00100000:3.0:1767319034.311015:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713344 00000100:00100000:3.0:1767319034.311055:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 601 00000100:00100000:3.0:1767319034.311063:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982713344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319034.311094:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319034.311144:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982713344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (321us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319034.311151:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 601 00000100:00100000:1.0:1767319034.311317:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982713344:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319034.769869:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319034.770151:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602432 00000100:00100000:1.0:1767319034.770161:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602432 00000100:00100000:1.0:1767319034.770439:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 874 00000100:00100000:1.0:1767319034.770450:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319034.770553:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (686us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319034.770561:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 874 00000100:00080000:2.0:1767319034.770673:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767319034.771396:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915602176 offset 0 length 520: 4 00000100:00100000:3.0:1767319034.771434:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319034.771449:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602304 00000100:00100000:1.0:1767319034.771455:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602304 00000100:00100000:1.0:1767319034.771489:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1146 00000100:00100000:1.0:1767319034.771496:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915602304:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319034.771520:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319034.771573:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915602304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 75us (138us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319034.772132:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1146 00010000:00010000:2.0:1767319035.012775:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319036.036443:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319037.061105:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319038.084691:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319038.917807:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319013/real 1767319013] req@ffff93463396e300 x1853163982707072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319038 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319038.917842:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463396e300 x1853163982707072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319038 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319038.917857:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319038.917881:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319038.917885:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982707072:0@lo:250:kworker.0 00000100:00080000:1.0:1767319038.918022:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319038.918028:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319038.918031:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319038.918035:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4261 00000100:00080000:1.0:1767319038.918039:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319038.918045:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319038.918078:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583800 x1853163982713600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319038.918144:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319038.918148:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319038.918150:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319038.918155:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4282 00000100:00080000:1.0:1767319038.918157:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319038.918161:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319038.918176:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582680 x1853163982713856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319038.918199:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319038.918202:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319038.918204:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319038.918208:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4282 00000100:00080000:1.0:1767319038.918210:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319038.918214:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319038.918227:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580a80 x1853163982714112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319038.918235:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319038.918238:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319038.918239:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319038.918242:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4282 00000100:00080000:1.0:1767319038.918244:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319038.918247:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319038.918257:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583b80 x1853163982714240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319038.918302:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319038.918305:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319038.918307:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319038.918311:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4282 00000100:00080000:1.0:1767319038.918313:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319038.918316:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319038.918326:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163982714624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319038.918351:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982713984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319038.918413:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319038.918472:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319038.918479:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982714368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319038.918505:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319038.918534:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713984 00000100:00100000:1.0:1767319038.918541:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713984 00000100:00100000:1.0:1767319038.918594:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 875 00000100:00100000:1.0:1767319038.918604:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982713984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319038.918664:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982713984:12345-0@lo:400:kworker.0 Request processed in 62us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319038.918672:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 875 00000100:00100000:1.0:1767319038.918694:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714368 00000100:00100000:1.0:1767319038.918698:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714368 00000100:00100000:1.0:1767319038.918723:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 561 00000100:00100000:1.0:1767319038.918729:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982714368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319038.918770:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982714368:12345-0@lo:400:kworker.0 Request processed in 30us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319038.918776:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 561 00000100:00080000:1.0:1767319038.918793:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319038.918797:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319038.918835:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982714368:0@lo:400:kworker.0 00010000:00080000:1.0:1767319038.918847:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a580e00 x1853163982713984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319054 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319038.918856:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982713984:0@lo:400:kworker.0 00000100:00100000:3.0:1767319038.919070:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982713728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319038.919097:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319038.919135:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319038.919141:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982714496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319038.919159:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.919217:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982713600:0@lo:250:kworker.0 00000400:00000100:0.0:1767319038.919238:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982713600 offset 0 length 520: 4 00000100:00100000:0.0:1767319038.919253:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982713856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319038.919283:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.919305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982714112:0@lo:38:kworker.0 00000100:00100000:0.0:1767319038.919321:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.919333:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982714240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319038.919351:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.919364:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982714624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319038.919378:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319038.919400:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714240 00000100:00100000:0.0:1767319038.919404:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714240 00000100:00100000:0.0:1767319038.919423:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 312 00000100:00100000:0.0:1767319038.919428:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982714240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319038.919444:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319038.919471:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982714240:12345-0@lo:38:kworker.0 Request processed in 41us (119us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319038.919476:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 312 00000100:00080000:0.0:1767319038.919500:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319038.919503:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319038.919506:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982714240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319038.919563:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713728 00000100:00100000:3.0:1767319038.919566:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713728 00000100:00100000:3.0:1767319038.919598:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982713856 00000100:00100000:3.0:1767319038.919600:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982713856 00000100:00100000:3.0:1767319038.919607:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714112 00000100:00100000:3.0:1767319038.919609:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714112 00000100:00100000:3.0:1767319038.919615:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714624 00000100:00100000:3.0:1767319038.919617:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714624 00000100:00100000:3.0:1767319038.919626:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 876 00000100:00100000:3.0:1767319038.919632:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982713728:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319038.919675:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982713728:12345-0@lo:400:kworker.0 Request processed in 43us (577us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319038.919684:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 876 00000100:00100000:3.0:1767319038.919697:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1147 00000100:00100000:3.0:1767319038.919701:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982713856:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319038.919711:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319038.919734:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982713856:12345-0@lo:38:kworker.0 Request processed in 31us (464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319038.919740:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1147 00000100:00100000:3.0:1767319038.919750:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148 00000100:00100000:3.0:1767319038.919753:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982714112:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767319038.919888:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319038.919914:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1149 00000100:00100000:0.0:1767319038.919919:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982714624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319038.919928:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319038.919950:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982714624:12345-0@lo:38:kworker.0 Request processed in 31us (571us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319038.919956:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1149 00000100:00080000:0.0:1767319038.919987:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319038.919991:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319038.919994:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982713856:0@lo:38:kworker.0 00000100:00080000:0.0:1767319038.920000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319038.920003:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319038.920006:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982714624:0@lo:38:kworker.0 00010000:00020000:3.0:1767319038.920049:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319038.920079:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982714112:12345-0@lo:38:kworker.0 Request processed in 324us (756us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319038.920084:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148 00010000:00080000:3.0:1767319038.920107:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581c00 x1853163982713728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319054 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319038.920117:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982713728:0@lo:400:kworker.0 00000100:00080000:0.0:1767319038.920193:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319038.920197:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319038.920201:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982714112:0@lo:38:kworker.0 00000100:00100000:1.0:1767319038.920967:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714496 00000100:00100000:1.0:1767319038.920975:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714496 00000100:00080000:0.0:1767319038.921017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319038.921072:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 562 00000100:00100000:1.0:1767319038.921081:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982714496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319038.921144:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982714496:12345-0@lo:400:kworker.0 Request processed in 64us (1982us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319038.921152:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 562 00000100:00100000:3.0:1767319038.921172:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982714496:0@lo:400:kworker.0 00010000:00010000:1.0:1767319039.109019:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319039.428688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982714752:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319039.428765:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319039.428815:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319039.428831:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982714880:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319039.428865:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319039.428973:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714752 00000100:00100000:2.0:1767319039.428981:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714752 00000100:00100000:2.0:1767319039.429033:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982714880 00000100:00100000:2.0:1767319039.429037:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982714880 00000100:00100000:2.0:1767319039.429053:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 602 00000100:00100000:2.0:1767319039.429061:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982714752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319039.429104:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319039.429182:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982714752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 122us (419us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319039.429189:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 602 00000100:00100000:2.0:1767319039.429205:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 603 00000100:00100000:2.0:1767319039.429211:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982714880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319039.429227:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319039.429256:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982714880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (392us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319039.429261:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 603 00000100:00100000:2.0:1767319039.429322:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982714880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319039.429499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982714752:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319039.883933:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319039.884078:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602560 00000100:00100000:0.0:1767319039.884087:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602560 00000100:00100000:0.0:1767319039.884165:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1150 00000100:00100000:0.0:1767319039.884172:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915602560:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319039.884191:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319039.884249:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915602560:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 77us (325us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319039.884254:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1150 00000100:00100000:0.0:1767319039.886130:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319039.886288:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602688 00000100:00100000:3.0:1767319039.886294:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602688 00000100:00100000:3.0:1767319039.886342:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 877 00000100:00100000:3.0:1767319039.886349:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602688:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319039.886393:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915602688:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 45us (271us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319039.886398:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 877 00000100:00080000:0.0:1767319039.886852:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319040.132525:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319041.156481:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319042.181577:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319043.205973:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319044.036396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319044.036407:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319044.036522:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767319044.036661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319044.036676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319044.036680:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319044.036689:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4287 00000100:00080000:3.0:1767319044.036694:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319044.036701:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319044.036727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374e00 x1853163982715136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319044.036779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319044.036782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319044.036784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319044.036788:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4287 00000100:00080000:3.0:1767319044.036790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319044.036794:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319044.036806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198bb80 x1853163982715392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319044.036815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319044.036818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319044.036821:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319044.036824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4287 00000100:00080000:3.0:1767319044.036826:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319044.036829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319044.036839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988380 x1853163982715520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319044.036905:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982715264:0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.036992:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.037062:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319044.037070:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982715648:0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.037097:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.037132:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715648 00000100:00100000:2.0:1767319044.037139:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715648 00000100:00100000:2.0:1767319044.037204:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 563 00000100:00100000:2.0:1767319044.037214:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982715648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.037267:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982715648:12345-0@lo:400:kworker.0 Request processed in 54us (168us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319044.037308:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 563 00000100:00100000:2.0:1767319044.037352:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982715648:0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.037388:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715264 00000100:00100000:2.0:1767319044.037392:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715264 00000100:00100000:2.0:1767319044.037425:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 878 00000100:00100000:2.0:1767319044.037432:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982715264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.037465:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982715264:12345-0@lo:400:kworker.0 Request processed in 35us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319044.037471:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 878 00010000:00080000:2.0:1767319044.037511:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988a80 x1853163982715264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319060 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319044.037521:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982715264:0@lo:400:kworker.0 00000100:00080000:0.0:1767319044.037601:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319044.037606:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319044.037619:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982715136:0@lo:38:kworker.0 00000100:00100000:0.0:1767319044.037651:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319044.037678:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982715392:0@lo:38:kworker.0 00000100:00100000:0.0:1767319044.037697:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319044.037711:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982715520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319044.037736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319044.037764:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715520 00000100:00100000:0.0:1767319044.037768:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715520 00000100:00100000:0.0:1767319044.037791:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 313 00000100:00100000:0.0:1767319044.037796:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982715520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319044.037814:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319044.037848:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982715520:12345-0@lo:38:kworker.0 Request processed in 51us (111us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319044.037854:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 313 00000100:00080000:0.0:1767319044.037883:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319044.037888:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319044.037891:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982715520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319044.038143:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715136 00000100:00100000:2.0:1767319044.038149:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715136 00000100:00100000:2.0:1767319044.038177:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715392 00000100:00100000:2.0:1767319044.038180:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715392 00000100:00100000:2.0:1767319044.038199:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151 00000100:00100000:2.0:1767319044.038204:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982715136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.038214:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.038249:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982715136:12345-0@lo:38:kworker.0 Request processed in 43us (596us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.038255:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151 00000100:00100000:2.0:1767319044.038267:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152 00000100:00100000:2.0:1767319044.038296:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982715392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319044.038305:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319044.038330:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982715392:12345-0@lo:38:kworker.0 Request processed in 33us (631us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319044.038338:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152 00000100:00080000:2.0:1767319044.038369:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.038374:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.038378:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982715136:0@lo:38:kworker.0 00000100:00080000:2.0:1767319044.038386:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319044.038389:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319044.038392:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982715392:0@lo:38:kworker.0 00000100:00100000:1.0:1767319044.039726:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982715776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.039782:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319044.039824:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982715008:0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.039846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319044.039957:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715776 00000100:00100000:2.0:1767319044.039965:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715776 00000100:00100000:2.0:1767319044.040023:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 564 00000100:00100000:2.0:1767319044.040040:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982715776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319044.040090:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982715776:12345-0@lo:400:kworker.0 Request processed in 60us (308us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319044.040097:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 564 00000100:00080000:2.0:1767319044.040139:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319044.041265:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982715776:0@lo:400:kworker.0 00000100:00080000:3.0:1767319044.041706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319044.041711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319044.041715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319044.041721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4287 00000100:00080000:3.0:1767319044.041724:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319044.041731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319044.041763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988700 x1853163982715904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319044.041821:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982715904:0@lo:38:kworker.0 00000100:00100000:3.0:1767319044.041875:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319044.043115:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715008 00000100:00100000:0.0:1767319044.043127:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715008 00000100:00080000:3.0:1767319044.043251:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319044.043485:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982715904 00000100:00100000:0.0:1767319044.043491:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982715904 00000100:00100000:1.0:1767319044.044964:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 879 00000100:00100000:1.0:1767319044.044999:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982715008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319044.045127:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982715008:12345-0@lo:400:kworker.0 Request processed in 141us (5277us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319044.045136:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 879 00000100:00100000:1.0:1767319044.045163:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1153 00000100:00100000:1.0:1767319044.045169:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982715904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319044.045635:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319044.045795:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982715904:12345-0@lo:38:kworker.0 Request processed in 622us (3916us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319044.045805:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1153 00000100:00080000:1.0:1767319044.045857:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319044.045863:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319044.045868:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982715904:0@lo:38:kworker.0 00010000:00080000:3.0:1767319044.046164:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604376680 x1853163982715008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319060 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319044.046382:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982715008:0@lo:400:kworker.0 00010000:00010000:2.0:1767319044.228810:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319044.548806:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982716032:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319044.548881:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319044.548994:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716032 00000100:00100000:0.0:1767319044.549004:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716032 00000100:00100000:0.0:1767319044.549071:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 604 00000100:00100000:0.0:1767319044.549081:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982716032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319044.549113:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319044.549164:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982716032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319044.549172:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 604 00000100:00100000:0.0:1767319044.549217:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982716032:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319044.549368:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319044.549378:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982716160:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319044.549404:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319044.549601:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716160 00000100:00100000:1.0:1767319044.549606:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716160 00000100:00100000:1.0:1767319044.549636:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 605 00000100:00100000:1.0:1767319044.549643:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982716160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319044.549752:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319044.549797:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982716160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 155us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319044.549804:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 605 00000100:00100000:0.0:1767319044.549871:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982716160:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319045.010924:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319045.011046:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319045.011303:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602816 00000100:00100000:2.0:1767319045.011313:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602816 00000100:00100000:2.0:1767319045.011361:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915602944 00000100:00100000:2.0:1767319045.011364:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915602944 00000100:00100000:2.0:1767319045.011423:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 880 00000100:00100000:2.0:1767319045.011433:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915602944:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319045.011508:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915602944:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319045.011517:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 880 00000100:00100000:2.0:1767319045.011535:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1154 00000100:00100000:2.0:1767319045.011540:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915602816:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319045.011559:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319045.011584:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915602816:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (670us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319045.011590:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1154 00000100:00080000:0.0:1767319045.011788:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319045.255967:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319046.281679:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319047.300684:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319048.325744:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319049.156593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319049.156605:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319049.156689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319049.156693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319049.156697:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319049.156707:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4292 00000100:00080000:3.0:1767319049.156712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319049.156719:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319049.156743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163982716416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319049.156785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319049.156789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319049.156791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319049.156795:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4292 00000100:00080000:3.0:1767319049.156798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319049.156802:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319049.156815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988700 x1853163982716672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319049.156824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319049.156827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319049.156829:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319049.156834:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4292 00000100:00080000:3.0:1767319049.156837:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319049.156842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319049.156854:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982716416:0@lo:38:kworker.0 00000100:00100000:3.0:1767319049.156856:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a300 x1853163982716800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319049.156890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319049.156894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319049.156896:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319049.156901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4292 00000100:00080000:3.0:1767319049.156903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319049.156907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319049.156919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988a80 x1853163982717184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319049.156970:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319049.157013:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982716672:0@lo:38:kworker.0 00000100:00100000:1.0:1767319049.157035:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319049.157056:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982716800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319049.157202:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319049.157238:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982717184:0@lo:38:kworker.0 00000100:00100000:1.0:1767319049.157258:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319049.157309:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716416 00000100:00100000:1.0:1767319049.157316:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716416 00000100:00100000:1.0:1767319049.157341:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716672 00000100:00100000:1.0:1767319049.157344:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716672 00000100:00100000:1.0:1767319049.157354:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717184 00000100:00100000:1.0:1767319049.157356:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717184 00000100:00100000:1.0:1767319049.157369:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1155 00000100:00100000:1.0:1767319049.157375:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982716416:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319049.157395:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319049.157433:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982716416:12345-0@lo:38:kworker.0 Request processed in 56us (463us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319049.157439:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1155 00000100:00100000:1.0:1767319049.157456:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1156 00000100:00100000:1.0:1767319049.157459:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982716672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319049.157467:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319049.157491:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982716672:12345-0@lo:38:kworker.0 Request processed in 29us (454us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319049.157497:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1156 00000100:00100000:1.0:1767319049.157507:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157 00000100:00100000:1.0:1767319049.157511:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319049.157533:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319049.157555:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717184:12345-0@lo:38:kworker.0 Request processed in 42us (296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319049.157560:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157 00000100:00080000:1.0:1767319049.157592:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319049.157596:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319049.157600:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982717184:0@lo:38:kworker.0 00000100:00080000:1.0:1767319049.157608:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319049.157612:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319049.157614:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982716416:0@lo:38:kworker.0 00000100:00080000:1.0:1767319049.157621:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319049.157624:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319049.157627:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982716672:0@lo:38:kworker.0 00000100:00100000:2.0:1767319049.158109:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982717056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319049.158145:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.158182:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319049.158188:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982716288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319049.158209:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319049.158532:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717056 00000100:00100000:3.0:1767319049.158539:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717056 00000100:00100000:3.0:1767319049.158612:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 565 00000100:00100000:3.0:1767319049.158620:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982717056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.158673:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982717056:12345-0@lo:400:kworker.0 Request processed in 55us (527us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319049.158680:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 565 00000100:00100000:0.0:1767319049.158698:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716800 00000100:00100000:0.0:1767319049.158704:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716800 00000100:00100000:3.0:1767319049.158731:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982716544:0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.158757:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319049.158777:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319049.158783:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982716928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.158804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319049.158826:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716928 00000100:00100000:3.0:1767319049.158829:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716928 00000100:00100000:3.0:1767319049.158867:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 566 00000100:00100000:3.0:1767319049.158872:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982716928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.158904:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982716928:12345-0@lo:400:kworker.0 Request processed in 32us (99us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319049.158910:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 566 00000100:00100000:3.0:1767319049.158953:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982716928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.158985:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716288 00000100:00100000:3.0:1767319049.158989:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716288 00000100:00100000:3.0:1767319049.159012:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982716544 00000100:00100000:3.0:1767319049.159015:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982716544 00000100:00100000:3.0:1767319049.159030:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 881 00000100:00100000:3.0:1767319049.159035:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982716288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.159064:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982716288:12345-0@lo:400:kworker.0 Request processed in 28us (854us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319049.159070:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 881 00000100:00100000:3.0:1767319049.159080:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 882 00000100:00100000:3.0:1767319049.159085:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982716544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.159110:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982716544:12345-0@lo:400:kworker.0 Request processed in 25us (352us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319049.159115:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 882 00010000:00080000:3.0:1767319049.159149:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988000 x1853163982716544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319065 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319049.159160:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982716544:0@lo:400:kworker.0 00000100:00100000:3.0:1767319049.159188:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 314 00000100:00100000:3.0:1767319049.159193:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982716800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319049.159208:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319049.159241:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982716800:12345-0@lo:38:kworker.0 Request processed in 46us (2037us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319049.159246:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 314 00000100:00080000:1.0:1767319049.160019:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319049.160027:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319049.160031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982716800:0@lo:38:kworker.0 00000100:00080000:2.0:1767319049.161146:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319049.161154:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319049.161157:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:2.0:1767319049.161194:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198b100 x1853163982716288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319065 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319049.161207:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982716288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319049.161223:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982717056:0@lo:400:kworker.0 00010000:00010000:1.0:1767319049.349735:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319049.668934:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319049.668955:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982717312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319049.669014:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319049.669205:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717312 00000100:00100000:0.0:1767319049.669213:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717312 00000100:00100000:0.0:1767319049.669267:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 606 00000100:00100000:0.0:1767319049.669294:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982717312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319049.669322:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319049.669390:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982717312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319049.669397:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 606 00000100:00100000:3.0:1767319049.669548:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982717312:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319049.669753:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982717440:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319049.669777:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319049.669880:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717440 00000100:00100000:2.0:1767319049.669885:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717440 00000100:00100000:2.0:1767319049.669910:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 607 00000100:00100000:2.0:1767319049.669915:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982717440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319049.669938:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319049.675141:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982717440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5215us (5352us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319049.675157:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 607 00000100:00100000:3.0:1767319049.675550:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982717440:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319050.123987:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319050.124233:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603072 00000100:00100000:2.0:1767319050.124247:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603072 00000100:00100000:2.0:1767319050.124343:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1158 00000100:00100000:2.0:1767319050.124363:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915603072:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319050.124385:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319050.124451:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915603072:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 99us (474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319050.124459:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1158 00000100:00100000:2.0:1767319050.125327:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319050.125466:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603200 00000100:00100000:1.0:1767319050.125472:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603200 00000100:00100000:1.0:1767319050.125537:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 883 00000100:00100000:1.0:1767319050.125546:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915603200:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319050.125610:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915603200:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 65us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319050.125619:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 883 00000100:00080000:2.0:1767319050.125789:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319050.373972:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319051.396556:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319052.423126:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319053.444578:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319054.277016:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319054.277027:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319054.277086:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319054.277091:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319054.277095:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319054.277103:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4297 00000100:00080000:0.0:1767319054.277108:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319054.277114:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319054.277137:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9f80 x1853163982717696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319054.277182:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319054.277185:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319054.277187:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319054.277192:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4297 00000100:00080000:0.0:1767319054.277194:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319054.277198:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319054.277210:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982717952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319054.277219:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319054.277222:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319054.277223:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982717696:0@lo:38:kworker.0 00000100:00080000:0.0:1767319054.277224:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319054.277227:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4297 00000100:00080000:0.0:1767319054.277229:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319054.277233:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319054.277243:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982718080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319054.277260:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319054.277263:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319054.277265:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319054.277268:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4297 00000100:00100000:2.0:1767319054.277318:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319054.277325:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982718336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319054.277357:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319054.277372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982717952:0@lo:38:kworker.0 00000100:00100000:1.0:1767319054.277379:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319054.277384:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982717568:0@lo:400:kworker.0 00000100:00100000:2.0:1767319054.277392:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319054.277441:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717696 00000100:00100000:3.0:1767319054.277449:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717696 00000100:00100000:1.0:1767319054.277488:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718336 00000100:00100000:1.0:1767319054.277493:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718336 00000100:00100000:2.0:1767319054.277503:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982718080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319054.277527:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319054.277554:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 567 00000100:00100000:1.0:1767319054.277563:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982718336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319054.277611:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982718336:12345-0@lo:400:kworker.0 Request processed in 47us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319054.277618:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 567 00000100:00080000:0.0:1767319054.284025:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319054.284036:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319054.284077:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982718464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319054.284113:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982718464:0@lo:38:kworker.0 00000100:00100000:0.0:1767319054.284117:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982718208:0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.284207:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717952 00000100:00100000:3.0:1767319054.284215:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717952 00000100:00100000:0.0:1767319054.284564:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.284599:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319054.284607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982717824:0@lo:400:kworker.0 00000100:00100000:0.0:1767319054.284631:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.291055:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718208 00000100:00100000:0.0:1767319054.291075:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718208 00000100:00100000:0.0:1767319054.291136:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 568 00000100:00100000:0.0:1767319054.291147:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982718208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319054.291221:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982718208:12345-0@lo:400:kworker.0 Request processed in 77us (6658us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319054.291228:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 568 00000100:00100000:0.0:1767319054.291291:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982718208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319054.291315:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718080 00000100:00100000:0.0:1767319054.291319:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718080 00000100:00100000:0.0:1767319054.291343:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 315 00000100:00100000:0.0:1767319054.291348:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982718080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319054.291365:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319054.291410:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982718080:12345-0@lo:38:kworker.0 Request processed in 61us (13882us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319054.291416:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 315 00000100:00080000:0.0:1767319054.291485:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319054.291489:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319054.291501:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319054.291809:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717824 00000100:00100000:1.0:1767319054.291815:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717824 00000100:00100000:1.0:1767319054.291862:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 884 00000100:00100000:1.0:1767319054.291870:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982717824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319054.291916:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982717824:12345-0@lo:400:kworker.0 Request processed in 47us (7283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319054.291922:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 884 00000100:00100000:1.0:1767319054.291940:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1159 00000100:00100000:1.0:1767319054.291944:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319054.291955:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319054.291979:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717696:12345-0@lo:38:kworker.0 Request processed in 34us (14661us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319054.292067:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1159 00000100:00100000:1.0:1767319054.292082:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160 00000100:00100000:1.0:1767319054.292087:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319054.292096:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319054.292122:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982717952:12345-0@lo:38:kworker.0 Request processed in 34us (14730us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319054.292128:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160 00010000:00080000:1.0:1767319054.292169:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fbb80 x1853163982717824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319070 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319054.292181:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982717824:0@lo:400:kworker.0 00000100:00080000:1.0:1767319054.292211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319054.292727:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982718336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.292771:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982717568 00000100:00100000:3.0:1767319054.292777:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982717568 00000100:00100000:3.0:1767319054.292906:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 885 00000100:00100000:3.0:1767319054.292915:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982717568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319054.292960:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982717568:12345-0@lo:400:kworker.0 Request processed in 48us (1458us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319054.292967:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 885 00000100:00080000:1.0:1767319054.293005:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319054.293036:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb800 x1853163982717568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319070 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319054.293046:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982717568:0@lo:400:kworker.0 00000100:00100000:2.0:1767319054.297077:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319054.297169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319054.297176:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319054.297181:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982717952:0@lo:38:kworker.0 00000100:00080000:2.0:1767319054.297195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319054.297199:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319054.297202:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982718080:0@lo:38:kworker.0 00000100:00080000:2.0:1767319054.297212:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319054.297216:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319054.297219:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982717696:0@lo:38:kworker.0 00000100:00100000:3.0:1767319054.297469:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718464 00000100:00100000:3.0:1767319054.297477:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718464 00000100:00100000:3.0:1767319054.297534:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1161 00000100:00100000:3.0:1767319054.297541:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982718464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319054.297561:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319054.297626:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982718464:12345-0@lo:38:kworker.0 Request processed in 83us (557us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319054.297634:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1161 00000100:00080000:2.0:1767319054.297779:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319054.297784:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319054.297787:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982718464:0@lo:38:kworker.0 00010000:00010000:1.0:1767319054.470454:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319054.788788:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982718592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319054.788876:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319054.789011:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718592 00000100:00100000:0.0:1767319054.789020:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718592 00000100:00100000:0.0:1767319054.789072:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 608 00000100:00100000:0.0:1767319054.789081:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982718592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319054.789110:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319054.789173:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982718592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319054.789182:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 608 00000100:00100000:0.0:1767319054.789224:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982718592:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319054.793011:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319054.793033:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982718720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319054.793099:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319054.793306:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718720 00000100:00100000:2.0:1767319054.793314:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718720 00000100:00100000:2.0:1767319054.793368:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 609 00000100:00100000:2.0:1767319054.793378:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982718720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319054.793416:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319054.793470:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982718720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (370us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319054.793477:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 609 00000100:00100000:1.0:1767319054.793961:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982718720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319055.254119:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319055.254166:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319055.254398:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603328 00000100:00100000:3.0:1767319055.254410:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603328 00000100:00100000:3.0:1767319055.254464:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603456 00000100:00100000:3.0:1767319055.254468:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603456 00000100:00100000:3.0:1767319055.254509:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 886 00000100:00100000:3.0:1767319055.254519:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915603456:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319055.254591:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915603456:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (424us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319055.254597:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 886 00000100:00100000:3.0:1767319055.254621:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1162 00000100:00100000:3.0:1767319055.254624:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915603328:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319055.254641:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319055.254657:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915603328:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 32us (551us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319055.254661:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1162 00000100:00080000:1.0:1767319055.254931:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319055.492912:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319056.516938:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319057.541251:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319058.564497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319059.396988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.396999:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319059.397070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.397074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.397077:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319059.397146:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319059.397164:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982718848:0@lo:400:kworker.0 00000100:00100000:0.0:1767319059.397231:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.397318:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718848 00000100:00100000:2.0:1767319059.397328:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718848 00000100:00100000:2.0:1767319059.397399:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 887 00000100:00100000:2.0:1767319059.397410:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982718848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319059.397467:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982718848:12345-0@lo:400:kworker.0 Request processed in 60us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319059.397474:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 887 00000100:00080000:0.0:1767319059.397542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319059.397575:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198bb80 x1853163982718848/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319075 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319059.397589:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982718848:0@lo:400:kworker.0 00000100:00080000:3.0:1767319059.397631:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.397635:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.397640:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319059.397660:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163982718976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319059.397715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.397719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.397721:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.397724:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.397727:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.397731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319059.397751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198bb80 x1853163982719232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319059.397760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.397763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.397765:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.397769:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.397772:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.397776:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319059.397787:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982719104:0@lo:400:kworker.0 00000100:00100000:3.0:1767319059.397791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988380 x1853163982719360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319059.397824:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319059.397826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319059.397830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319059.397832:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319059.397837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4302 00000100:00080000:3.0:1767319059.397840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319059.397843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319059.397851:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319059.397859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163982719744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319059.397861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982719488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319059.397894:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319059.397920:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719104 00000100:00100000:3.0:1767319059.397926:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719104 00000100:00100000:0.0:1767319059.397927:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982719616:0@lo:400:kworker.0 00000100:00100000:0.0:1767319059.397957:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319059.397968:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 888 00000100:00100000:3.0:1767319059.397976:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982719104:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319059.397988:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319059.398018:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982719104:12345-0@lo:400:kworker.0 Request processed in 43us (193us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319059.398026:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 888 00010000:00080000:0.0:1767319059.398035:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988a80 x1853163982719104/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319075 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319059.398046:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982719104:0@lo:400:kworker.0 00000100:00100000:2.0:1767319059.398305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982718976:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.398345:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.398367:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982719232:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.398396:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.398413:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982719360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.398448:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.398472:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982719744:0@lo:38:kworker.0 00000100:00100000:2.0:1767319059.398490:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319059.398519:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719360 00000100:00100000:2.0:1767319059.398523:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719360 00000100:00100000:2.0:1767319059.398544:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 316 00000100:00100000:2.0:1767319059.398549:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982719360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.398566:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.398599:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982719360:12345-0@lo:38:kworker.0 Request processed in 49us (163us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.398606:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 316 00000100:00100000:2.0:1767319059.398624:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982718976 00000100:00100000:2.0:1767319059.398627:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982718976 00000100:00100000:2.0:1767319059.398642:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719232 00000100:00100000:2.0:1767319059.398645:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719232 00000100:00100000:2.0:1767319059.398653:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719744 00000100:00100000:2.0:1767319059.398655:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719744 00000100:00100000:2.0:1767319059.398664:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163 00000100:00100000:2.0:1767319059.398669:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982718976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.398693:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.398719:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982718976:12345-0@lo:38:kworker.0 Request processed in 48us (373us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.398725:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163 00000100:00100000:2.0:1767319059.398736:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164 00000100:00100000:2.0:1767319059.398740:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982719232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.398765:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.398788:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982719232:12345-0@lo:38:kworker.0 Request processed in 46us (391us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.398793:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164 00000100:00100000:2.0:1767319059.398804:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1165 00000100:00100000:2.0:1767319059.398807:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982719744:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319059.398815:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319059.398835:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982719744:12345-0@lo:38:kworker.0 Request processed in 26us (344us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319059.398840:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1165 00000100:00080000:2.0:1767319059.398866:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.398871:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.398874:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982718976:0@lo:38:kworker.0 00000100:00080000:2.0:1767319059.398882:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.398887:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.398890:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982719232:0@lo:38:kworker.0 00000100:00080000:2.0:1767319059.398898:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.398901:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.398904:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982719360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319059.398911:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319059.398913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319059.398915:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982719744:0@lo:38:kworker.0 00000100:00100000:1.0:1767319059.399966:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719488 00000100:00100000:1.0:1767319059.399975:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719488 00000100:00100000:1.0:1767319059.400038:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719616 00000100:00100000:1.0:1767319059.400042:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719616 00000100:00100000:1.0:1767319059.400081:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 569 00000100:00100000:1.0:1767319059.400090:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982719488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319059.400157:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982719488:12345-0@lo:400:kworker.0 Request processed in 69us (2262us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319059.400164:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 569 00000100:00100000:1.0:1767319059.400179:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 570 00000100:00100000:1.0:1767319059.400184:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982719616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319059.400213:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982719616:12345-0@lo:400:kworker.0 Request processed in 29us (2256us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319059.400219:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 570 00000100:00100000:1.0:1767319059.400261:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982719488:0@lo:400:kworker.0 00000100:00080000:0.0:1767319059.400356:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319059.400360:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319059.400390:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982719616:0@lo:400:kworker.0 00010000:00010000:2.0:1767319059.588730:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319059.908712:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982719872:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319059.908799:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319059.909730:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319059.909752:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982720000:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319059.909792:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319059.911795:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982719872 00000100:00100000:3.0:1767319059.911809:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982719872 00000100:00100000:3.0:1767319059.911901:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720000 00000100:00100000:3.0:1767319059.911905:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720000 00000100:00100000:2.0:1767319059.911942:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 610 00000100:00100000:2.0:1767319059.911955:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982719872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319059.911989:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319059.912077:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982719872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 124us (3277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319059.912085:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 610 00000100:00100000:2.0:1767319059.912103:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 611 00000100:00100000:2.0:1767319059.912110:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982720000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319059.912124:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319059.912155:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982720000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (2364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319059.912174:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 611 00000100:00100000:0.0:1767319059.912254:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982719872:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319059.912320:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982720000:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319060.364862:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319060.364947:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603712 00000100:00100000:1.0:1767319060.364959:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603712 00000100:00100000:1.0:1767319060.365088:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 889 00000100:00100000:1.0:1767319060.365100:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915603712:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319060.365178:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915603712:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (325us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319060.365186:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 889 00000100:00080000:0.0:1767319060.365239:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319060.366833:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319060.366893:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603584 00000100:00100000:1.0:1767319060.366901:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603584 00000100:00100000:1.0:1767319060.366993:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1166 00000100:00100000:1.0:1767319060.367001:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915603584:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319060.367025:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319060.367086:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915603584:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 84us (257us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319060.367091:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1166 00000400:00000100:2.0:1767319060.376520:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915603840 offset 0 length 520: 4 00010000:00010000:1.0:1767319060.616027:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319061.636492:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319062.661544:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319063.492523:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319038/real 1767319038] req@ffff93460a583800 x1853163982713600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319063 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319063.492572:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a583800 x1853163982713600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319063 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319063.492600:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319063.492630:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319063.492634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982713600:0@lo:250:kworker.0 00000100:00080000:3.0:1767319063.492803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319063.492827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319063.492830:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319063.492835:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4287 00000100:00080000:3.0:1767319063.492839:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319063.492846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319063.492879:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982720128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319063.492974:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982720256:0@lo:400:kworker.0 00000100:00100000:0.0:1767319063.493052:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319063.493187:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720256 00000100:00100000:1.0:1767319063.493195:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720256 00000100:00100000:1.0:1767319063.493312:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 890 00000100:00100000:1.0:1767319063.493324:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982720256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.493422:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982720128:0@lo:250:kworker.0 00000100:00080000:3.0:1767319063.494303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319063.494310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319063.494313:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319063.494317:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4307 00000100:00080000:3.0:1767319063.494320:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319063.494325:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319063.494351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2680 x1853163982720384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319063.494381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319063.494385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319063.494386:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319063.494390:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4307 00000100:00080000:3.0:1767319063.494392:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319063.494396:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319063.494406:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3b80 x1853163982720640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319063.494414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319063.494417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319063.494419:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319063.494422:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4307 00000100:00080000:3.0:1767319063.494425:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319063.494428:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319063.494437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1180 x1853163982720768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:2.0:1767319063.494548:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982720128 offset 0 length 520: 4 00000100:00100000:2.0:1767319063.494593:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982720384:0@lo:38:kworker.0 00000100:00100000:2.0:1767319063.494627:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.494645:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982720640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319063.494694:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.494713:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982720768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319063.494736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.494849:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982720512:0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.494874:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.494894:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319063.494900:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982720896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.494921:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.494961:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720768 00000100:00100000:2.0:1767319063.494967:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720768 00000100:00100000:2.0:1767319063.494995:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 317 00000100:00100000:2.0:1767319063.495002:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982720768:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319063.495022:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319063.495057:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982720768:12345-0@lo:38:kworker.0 Request processed in 54us (318us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319063.495063:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 317 00000100:00100000:2.0:1767319063.495084:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720384 00000100:00100000:2.0:1767319063.495087:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720384 00000100:00100000:2.0:1767319063.495100:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720640 00000100:00100000:2.0:1767319063.495102:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720640 00000100:00100000:2.0:1767319063.495109:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720512 00000100:00100000:2.0:1767319063.495111:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720512 00000100:00100000:2.0:1767319063.495129:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 891 00000100:00100000:2.0:1767319063.495134:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982720512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.495170:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982720512:12345-0@lo:400:kworker.0 Request processed in 36us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319063.495175:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 891 00000100:00100000:2.0:1767319063.495187:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167 00000100:00100000:2.0:1767319063.495190:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982720384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319063.495198:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319063.495219:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982720384:12345-0@lo:38:kworker.0 Request processed in 28us (592us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319063.495224:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167 00000100:00100000:2.0:1767319063.495233:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1168 00000100:00100000:2.0:1767319063.495236:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982720640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319063.495244:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319063.495263:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982720640:12345-0@lo:38:kworker.0 Request processed in 26us (570us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319063.495269:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1168 00000100:00100000:2.0:1767319063.495307:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982720896 00000100:00100000:2.0:1767319063.495311:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982720896 00000100:00100000:2.0:1767319063.495337:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 571 00000100:00100000:2.0:1767319063.495343:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982720896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.495373:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982720896:12345-0@lo:400:kworker.0 Request processed in 29us (450us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319063.495378:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 571 00000100:00100000:2.0:1767319063.495417:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982720896:0@lo:400:kworker.0 00010000:00080000:2.0:1767319063.495429:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0a80 x1853163982720512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319079 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319063.495437:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982720512:0@lo:400:kworker.0 00000100:00080000:2.0:1767319063.495615:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319063.495620:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319063.495624:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982720384:0@lo:38:kworker.0 00000100:00080000:2.0:1767319063.495632:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319063.495635:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319063.495638:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982720640:0@lo:38:kworker.0 00000100:00080000:2.0:1767319063.495646:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319063.495648:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319063.495651:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982720768:0@lo:38:kworker.0 00000100:00100000:2.0:1767319063.495705:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982721024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.495731:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319063.495757:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721024 00000100:00100000:2.0:1767319063.495761:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721024 00000100:00100000:2.0:1767319063.496392:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 572 00000100:00100000:2.0:1767319063.496401:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982721024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319063.496591:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982721024:12345-0@lo:400:kworker.0 Request processed in 45us (712us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319063.496601:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 572 00000100:00100000:2.0:1767319063.496634:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982721024:0@lo:400:kworker.0 00000100:00080000:2.0:1767319063.496688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319063.496692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319063.496694:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319063.497996:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982720256:12345-0@lo:400:kworker.0 Request processed in 4669us (4937us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319063.498009:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 890 00010000:00080000:0.0:1767319063.504915:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1c00 x1853163982720256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319079 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319063.504941:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982720256:0@lo:400:kworker.0 00000100:00080000:3.0:1767319063.505858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319063.505865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319063.505870:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319063.505876:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4307 00000100:00080000:3.0:1767319063.505881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319063.505889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319063.505934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1500 x1853163982721152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319063.506378:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982721152:0@lo:38:kworker.0 00000100:00100000:2.0:1767319063.506528:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319063.514634:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721152 00000100:00100000:1.0:1767319063.514654:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721152 00000100:00100000:1.0:1767319063.514736:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169 00000100:00100000:1.0:1767319063.514746:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982721152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319063.514780:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319063.514890:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982721152:12345-0@lo:38:kworker.0 Request processed in 144us (8361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319063.514898:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169 00000100:00080000:3.0:1767319063.515173:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319063.515179:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319063.515183:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982721152:0@lo:38:kworker.0 00010000:00010000:1.0:1767319063.684644:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319064.708611:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319065.028520:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319065.028543:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982721280:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319065.028622:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319065.028731:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721280 00000100:00100000:0.0:1767319065.028739:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721280 00000100:00100000:0.0:1767319065.028820:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 612 00000100:00100000:0.0:1767319065.028831:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982721280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319065.028908:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319065.028948:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982721280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319065.028953:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 612 00000100:00100000:0.0:1767319065.028996:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982721280:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319065.029059:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319065.029613:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319065.029624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982721408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319065.029672:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319065.029701:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721408 00000100:00100000:0.0:1767319065.029706:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721408 00000100:00100000:0.0:1767319065.029737:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 613 00000100:00100000:0.0:1767319065.029744:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982721408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319065.029780:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319065.029826:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982721408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (153us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319065.029832:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 613 00000100:00100000:0.0:1767319065.029859:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982721408:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319065.029936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319065.484926:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319065.484971:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319065.485117:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604096 00000100:00100000:1.0:1767319065.485130:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604096 00000100:00100000:2.0:1767319065.485212:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915603968 00000100:00100000:2.0:1767319065.485219:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915603968 00000100:00100000:1.0:1767319065.485232:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 892 00000100:00080000:3.0:1767319065.485234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319065.485355:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915604096:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319065.485435:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915604096:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 193us (520us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319065.485591:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1170 00000100:00100000:2.0:1767319065.485598:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915603968:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319065.485617:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319065.485665:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915603968:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 67us (694us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319065.485672:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1170 00000100:00100000:1.0:1767319065.486524:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 892 00010000:00010000:2.0:1767319065.733190:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319066.756495:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319067.780619:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319068.612491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319068.612503:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319068.612613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319068.612617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319068.612621:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319068.612628:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4312 00000100:00080000:0.0:1767319068.612633:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319068.612639:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319068.612666:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddf80 x1853163982721664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319068.612708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319068.612712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319068.612714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319068.612718:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4312 00000100:00080000:0.0:1767319068.612721:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319068.612725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319068.612737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc700 x1853163982721920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319068.612745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319068.612749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319068.612750:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319068.612754:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4312 00000100:00080000:0.0:1767319068.612756:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319068.612760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319068.612771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163982722048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319068.612789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319068.612793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319068.612795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319068.612798:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4312 00000100:00080000:0.0:1767319068.612800:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319068.612803:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319068.612812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddc00 x1853163982722432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319068.612842:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982721792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319068.612947:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319068.612983:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319068.612989:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982722176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319068.613031:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319068.613118:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982721664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319068.613149:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319068.613174:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982721920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319068.613193:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319068.613224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982722048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319068.613246:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319068.613262:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982722432:0@lo:38:kworker.0 00000100:00100000:3.0:1767319068.613303:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319068.613359:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722048 00000100:00100000:3.0:1767319068.613367:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722048 00000100:00100000:3.0:1767319068.613401:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 318 00000100:00100000:3.0:1767319068.613407:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982722048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319068.613427:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319068.613468:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982722048:12345-0@lo:38:kworker.0 Request processed in 59us (220us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319068.613475:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 318 00000100:00100000:3.0:1767319068.613497:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721792 00000100:00100000:3.0:1767319068.613501:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721792 00000100:00100000:3.0:1767319068.613539:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721664 00000100:00100000:3.0:1767319068.613542:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721664 00000100:00100000:3.0:1767319068.613550:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721920 00000100:00100000:3.0:1767319068.613553:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721920 00000100:00100000:3.0:1767319068.613560:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722432 00000100:00100000:3.0:1767319068.613562:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722432 00000100:00100000:3.0:1767319068.613583:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 893 00000100:00100000:3.0:1767319068.613591:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982721792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319068.613636:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982721792:12345-0@lo:400:kworker.0 Request processed in 47us (690us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319068.613643:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 893 00000100:00100000:3.0:1767319068.613656:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1171 00000100:00100000:3.0:1767319068.613659:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982721664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319068.613669:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319068.613691:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982721664:12345-0@lo:38:kworker.0 Request processed in 30us (542us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319068.613697:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1171 00000100:00100000:3.0:1767319068.613708:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172 00000100:00100000:3.0:1767319068.613711:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982721920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319068.613720:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319068.613741:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982721920:12345-0@lo:38:kworker.0 Request processed in 28us (547us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319068.613747:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172 00000100:00100000:3.0:1767319068.613758:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173 00000100:00100000:3.0:1767319068.613762:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982722432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319068.613770:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319068.613791:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982722432:12345-0@lo:38:kworker.0 Request processed in 28us (487us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319068.613797:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173 00000100:00080000:3.0:1767319068.613815:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319068.613845:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319068.613850:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319068.613853:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982721664:0@lo:38:kworker.0 00000100:00080000:3.0:1767319068.613863:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319068.613866:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319068.613869:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982721920:0@lo:38:kworker.0 00000100:00080000:3.0:1767319068.613877:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319068.613880:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319068.613882:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982722048:0@lo:38:kworker.0 00000100:00080000:3.0:1767319068.613889:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319068.613892:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319068.613895:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982722432:0@lo:38:kworker.0 00010000:00080000:3.0:1767319068.613928:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc380 x1853163982721792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319084 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319068.613938:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982721792:0@lo:400:kworker.0 00000100:00100000:2.0:1767319068.614014:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982721536:0@lo:400:kworker.0 00000100:00100000:2.0:1767319068.614042:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319068.614067:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319068.614072:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982722304:0@lo:400:kworker.0 00000100:00100000:2.0:1767319068.614094:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319068.614120:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722176 00000100:00100000:2.0:1767319068.614124:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722176 00000100:00100000:2.0:1767319068.614153:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722304 00000100:00100000:2.0:1767319068.614155:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722304 00000100:00100000:2.0:1767319068.614170:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 573 00000100:00100000:2.0:1767319068.614175:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982722176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319068.614204:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982722176:12345-0@lo:400:kworker.0 Request processed in 29us (1173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319068.614209:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 573 00000100:00100000:2.0:1767319068.614219:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 574 00000100:00100000:2.0:1767319068.614224:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982722304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319068.614248:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982722304:12345-0@lo:400:kworker.0 Request processed in 24us (154us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319068.614252:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 574 00000100:00080000:2.0:1767319068.614288:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319068.614291:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319068.614304:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982722304:0@lo:400:kworker.0 00000100:00100000:1.0:1767319068.614400:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982721536 00000100:00100000:1.0:1767319068.614404:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982721536 00000100:00100000:1.0:1767319068.614439:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 894 00000100:00100000:1.0:1767319068.614445:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982721536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319068.614478:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982721536:12345-0@lo:400:kworker.0 Request processed in 33us (435us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319068.614483:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 894 00010000:00080000:1.0:1767319068.614508:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988a80 x1853163982721536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319084 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319068.614516:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982721536:0@lo:400:kworker.0 00000100:00080000:2.0:1767319068.614626:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319068.614772:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982722176:0@lo:400:kworker.0 00010000:00010000:2.0:1767319068.804510:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319069.828720:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319070.148717:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319070.148743:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982722560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319070.148817:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319070.148872:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722560 00000100:00100000:1.0:1767319070.148881:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722560 00000100:00100000:1.0:1767319070.148947:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 614 00000100:00100000:1.0:1767319070.148956:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982722560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319070.148984:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319070.149047:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982722560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (225us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319070.149055:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 614 00000100:00080000:3.0:1767319070.149119:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319070.149192:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982722560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319070.150330:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982722688:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319070.150386:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319070.150444:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722688 00000100:00100000:0.0:1767319070.150451:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722688 00000100:00100000:0.0:1767319070.150587:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 615 00000100:00100000:0.0:1767319070.150597:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982722688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319070.151018:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319070.151083:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982722688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 488us (696us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319070.151092:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 615 00000100:00100000:2.0:1767319070.151229:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982722688:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319070.151572:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319070.608886:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319070.608932:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319070.609031:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604352 00000100:00100000:3.0:1767319070.609042:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604352 00000100:00100000:3.0:1767319070.609115:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604224 00000100:00100000:3.0:1767319070.609119:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604224 00000100:00100000:3.0:1767319070.609136:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 895 00000100:00100000:3.0:1767319070.609147:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604352:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319070.609222:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604352:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (345us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319070.609229:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 895 00000100:00100000:3.0:1767319070.609247:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1174 00000100:00100000:3.0:1767319070.609252:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915604224:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319070.609307:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319070.609334:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915604224:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 79us (401us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319070.609340:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1174 00000100:00080000:0.0:1767319070.609733:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319070.852865:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319071.876515:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319072.900516:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319073.736089:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319073.736101:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319073.736163:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319073.736175:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319073.736180:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319073.736189:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4317 00000100:00080000:0.0:1767319073.736194:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319073.736210:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319073.736231:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982722816:0@lo:400:kworker.0 00000100:00080000:0.0:1767319073.736248:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319073.736302:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982722944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319073.736347:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319073.736358:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319073.736363:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319073.736365:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319073.736370:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4317 00000100:00080000:0.0:1767319073.736372:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319073.736376:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319073.736389:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982723200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319073.736397:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319073.736401:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319073.736403:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319073.736407:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4317 00000100:00100000:1.0:1767319073.736409:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722816 00000100:00080000:0.0:1767319073.736410:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319073.736416:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319073.736419:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722816 00000100:00100000:0.0:1767319073.736429:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163982723328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319073.736459:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319073.736463:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319073.736465:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319073.736469:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4317 00000100:00080000:0.0:1767319073.736472:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319073.736492:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896 00000100:00080000:0.0:1767319073.736502:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319073.736505:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982722816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319073.736520:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163982723712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319073.736526:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982722944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319073.736555:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982723072:0@lo:400:kworker.0 00000100:00100000:1.0:1767319073.736557:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982722816:12345-0@lo:400:kworker.0 Request processed in 57us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319073.736557:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319073.736569:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896 00000100:00100000:3.0:1767319073.736578:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982723200:0@lo:38:kworker.0 00000100:00100000:1.0:1767319073.736585:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982722944 00000100:00100000:1.0:1767319073.736588:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982722944 00000100:00100000:0.0:1767319073.736599:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319073.736904:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723072 00000100:00100000:1.0:1767319073.736910:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723072 00000100:00100000:1.0:1767319073.736933:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 897 00000100:00100000:1.0:1767319073.736939:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982723072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319073.736986:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319073.736994:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982723584:0@lo:400:kworker.0 00000100:00100000:1.0:1767319073.737230:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982723072:12345-0@lo:400:kworker.0 Request processed in 290us (631us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319073.737238:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 897 00000100:00100000:1.0:1767319073.737254:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175 00000100:00100000:1.0:1767319073.737258:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982722944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319073.737318:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319073.737345:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982722944:12345-0@lo:38:kworker.0 Request processed in 85us (789us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319073.737351:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175 00010000:00080000:1.0:1767319073.737407:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dea00 x1853163982722816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319089 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319073.737420:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982722816:0@lo:400:kworker.0 00000100:00080000:1.0:1767319073.737454:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319073.737469:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982723456:0@lo:400:kworker.0 00000100:00100000:1.0:1767319073.737505:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319073.737533:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723456 00000100:00100000:1.0:1767319073.737536:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723456 00000100:00100000:1.0:1767319073.737565:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 575 00000100:00100000:1.0:1767319073.737570:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982723456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319073.737603:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982723456:12345-0@lo:400:kworker.0 Request processed in 32us (97us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319073.737609:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 575 00000100:00080000:1.0:1767319073.737622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319073.737638:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982723456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319073.744593:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319073.745183:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723584 00000100:00100000:2.0:1767319073.745192:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723584 00010000:00080000:0.0:1767319073.745424:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dd180 x1853163982723072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319089 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319073.745440:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982723072:0@lo:400:kworker.0 00000100:00100000:0.0:1767319073.745516:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 576 00000100:00100000:0.0:1767319073.745528:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982723584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319073.745589:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982723584:12345-0@lo:400:kworker.0 Request processed in 63us (1002us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319073.745596:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 576 00000100:00100000:2.0:1767319073.745802:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982723584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319073.745836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319073.745867:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982723328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319073.746238:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319073.746413:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319073.746441:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723200 00000100:00100000:0.0:1767319073.746448:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723200 00000100:00100000:0.0:1767319073.746490:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176 00000100:00100000:0.0:1767319073.746496:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982723200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319073.746518:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319073.746557:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982723200:12345-0@lo:38:kworker.0 Request processed in 60us (720us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319073.746564:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176 00000100:00100000:3.0:1767319073.746755:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723328 00000100:00100000:3.0:1767319073.746761:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723328 00000100:00100000:2.0:1767319073.746770:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982723712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319073.746801:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319073.746829:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319073.746837:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319073.746838:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 319 00000100:00100000:2.0:1767319073.746844:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982722944:0@lo:38:kworker.0 00000100:00100000:3.0:1767319073.746849:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982723328:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767319073.746858:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:3.0:1767319073.746864:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319073.746865:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319073.746900:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982723328:12345-0@lo:38:kworker.0 Request processed in 52us (660us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319073.746908:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 319 00000100:00100000:3.0:1767319073.746925:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723712 00000100:00100000:3.0:1767319073.746930:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723712 00000100:00100000:3.0:1767319073.746951:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1177 00000100:00100000:3.0:1767319073.746955:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982723712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319073.746964:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319073.747003:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982723712:12345-0@lo:38:kworker.0 Request processed in 46us (201us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319073.747009:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1177 00000100:00100000:2.0:1767319073.747522:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982723200:0@lo:38:kworker.0 00000100:00080000:2.0:1767319073.747543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319073.747547:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319073.747549:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982723328:0@lo:38:kworker.0 00000100:00080000:2.0:1767319073.747575:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319073.747578:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319073.747580:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982723712:0@lo:38:kworker.0 00010000:00010000:1.0:1767319073.925635:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319074.948573:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319075.268701:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982723840:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319075.268779:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319075.269061:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723840 00000100:00100000:0.0:1767319075.269070:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723840 00000100:00100000:0.0:1767319075.269148:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 616 00000100:00100000:0.0:1767319075.269158:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982723840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319075.269186:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319075.269235:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982723840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (455us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319075.269241:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 616 00000100:00100000:0.0:1767319075.269320:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982723840:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319075.269439:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319075.270912:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982723968:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319075.270969:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319075.272021:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982723968 00000100:00100000:3.0:1767319075.272099:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982723968 00000100:00100000:3.0:1767319075.272239:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 617 00000100:00100000:3.0:1767319075.272249:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982723968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319075.272302:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767319075.272817:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319075.273389:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982723968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1138us (2416us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319075.273399:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 617 00000100:00100000:0.0:1767319075.274015:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982723968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319075.724673:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319075.724858:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319075.730297:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604608 00000100:00100000:0.0:1767319075.730319:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604608 00000100:00100000:0.0:1767319075.730434:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604480 00000100:00100000:0.0:1767319075.730449:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604480 00000100:00080000:3.0:1767319075.730560:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319075.730638:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 898 00000100:00100000:0.0:1767319075.730650:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604608:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319075.730740:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604608:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (6079us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319075.730748:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 898 00000100:00100000:0.0:1767319075.730769:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1178 00000100:00100000:0.0:1767319075.730774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915604480:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319075.730793:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319075.730823:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915604480:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (5965us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319075.730830:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1178 00010000:00010000:1.0:1767319075.973605:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319076.996591:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319078.020675:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319078.853159:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319078.853170:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319078.853297:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319078.853302:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319078.853306:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319078.853315:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4322 00000100:00080000:0.0:1767319078.853319:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319078.853326:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319078.853365:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163982724224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319078.853411:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319078.853413:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982724096:0@lo:400:kworker.0 00000100:00080000:0.0:1767319078.853415:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319078.853419:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319078.853424:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4322 00000100:00080000:0.0:1767319078.853426:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319078.853439:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319078.853451:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982724480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319078.853461:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319078.853464:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319078.853465:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319078.853469:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4322 00000100:00080000:0.0:1767319078.853471:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319078.853481:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319078.853491:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982724608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319078.853512:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319078.853515:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319078.853517:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319078.853523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319078.853526:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4322 00000100:00080000:0.0:1767319078.853529:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319078.853532:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319078.853604:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319078.853610:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982724864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319078.853648:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319078.853681:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724096 00000100:00100000:2.0:1767319078.853688:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724096 00000100:00100000:0.0:1767319078.853702:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dea00 x1853163982724992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319078.853735:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982724736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319078.853750:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 899 00000100:00100000:2.0:1767319078.853768:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982724096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319078.853782:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319078.853812:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319078.853834:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982724096:12345-0@lo:400:kworker.0 Request processed in 75us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319078.853842:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899 00000100:00100000:2.0:1767319078.853863:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724864 00000100:00100000:2.0:1767319078.853866:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724864 00000100:00100000:2.0:1767319078.853889:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724736 00000100:00100000:2.0:1767319078.853892:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724736 00000100:00100000:2.0:1767319078.853926:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 577 00000100:00100000:2.0:1767319078.853935:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982724864:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319078.853936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319078.853941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319078.853943:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319078.853959:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982724224:0@lo:38:kworker.0 00000100:00100000:2.0:1767319078.853978:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982724864:12345-0@lo:400:kworker.0 Request processed in 44us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319078.853985:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 577 00000100:00100000:2.0:1767319078.853997:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 578 00000100:00100000:2.0:1767319078.854005:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982724736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319078.854016:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319078.854047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982724480:0@lo:38:kworker.0 00000100:00100000:2.0:1767319078.854054:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982724736:12345-0@lo:400:kworker.0 Request processed in 50us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319078.854060:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 578 00000100:00100000:3.0:1767319078.854076:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319078.854107:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982724864:0@lo:400:kworker.0 00000100:00100000:0.0:1767319078.854109:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982724352:0@lo:400:kworker.0 00000100:00100000:1.0:1767319078.854167:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724224 00000100:00100000:1.0:1767319078.854172:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724224 00000100:00100000:1.0:1767319078.854207:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724480 00000100:00100000:1.0:1767319078.854210:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724480 00000100:00100000:1.0:1767319078.854226:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179 00000100:00100000:1.0:1767319078.854231:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982724224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319078.854248:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319078.860825:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982724608:0@lo:38:kworker.0 00000100:00100000:3.0:1767319078.860954:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319078.860993:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982724992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319078.861022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319078.861063:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724608 00000100:00100000:3.0:1767319078.861070:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724608 00000100:00100000:3.0:1767319078.861103:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 320 00000100:00100000:3.0:1767319078.861111:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982724608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319078.861131:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319078.861169:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982724608:12345-0@lo:38:kworker.0 Request processed in 58us (216us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319078.861176:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 320 00000100:00100000:3.0:1767319078.861201:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724992 00000100:00100000:3.0:1767319078.861204:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724992 00000100:00100000:3.0:1767319078.861217:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1180 00000100:00100000:3.0:1767319078.861222:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982724480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319078.861233:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319078.861255:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982724480:12345-0@lo:38:kworker.0 Request processed in 32us (7182us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319078.861261:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1180 00000100:00100000:3.0:1767319078.861288:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181 00000100:00100000:3.0:1767319078.861292:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982724992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319078.861300:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319078.861318:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982724992:12345-0@lo:38:kworker.0 Request processed in 25us (296us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319078.861323:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181 00000100:00080000:3.0:1767319078.861355:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319078.861360:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319078.861364:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982724480:0@lo:38:kworker.0 00000100:00080000:3.0:1767319078.861371:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319078.861374:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319078.861387:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982724608:0@lo:38:kworker.0 00000100:00080000:3.0:1767319078.861395:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319078.861399:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319078.861401:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982724992:0@lo:38:kworker.0 00010000:00080000:2.0:1767319078.861684:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddf80 x1853163982724096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319094 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319078.861697:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982724096:0@lo:400:kworker.0 00000100:00100000:0.0:1767319078.862372:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319078.862428:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982724736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319078.862507:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982724352 00000100:00100000:3.0:1767319078.862512:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982724352 00000100:00100000:3.0:1767319078.862570:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900 00000100:00100000:3.0:1767319078.862576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982724352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319078.862757:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982724352:12345-0@lo:400:kworker.0 Request processed in 181us (386us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319078.862764:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900 00010000:00080000:2.0:1767319078.862842:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc000 x1853163982724352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319094 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319078.862859:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982724352:0@lo:400:kworker.0 00000100:00080000:0.0:1767319078.863023:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319078.863640:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982724224:12345-0@lo:38:kworker.0 Request processed in 9404us (9622us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319078.863648:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179 00000100:00080000:3.0:1767319078.863736:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319078.863741:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319078.863744:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982724224:0@lo:38:kworker.0 00010000:00010000:1.0:1767319079.045177:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319080.071056:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319080.388951:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319080.388975:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982725248:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319080.389047:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319080.389178:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319080.389239:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982725120:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319080.389309:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319080.389379:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725248 00000100:00100000:3.0:1767319080.389388:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725248 00000100:00100000:3.0:1767319080.389451:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725120 00000100:00100000:1.0:1767319080.389558:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 618 00000100:00100000:1.0:1767319080.389571:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982725248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319080.389604:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319080.389652:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982725248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (604us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319080.389659:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 618 00000100:00100000:1.0:1767319080.389706:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982725248:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319080.389962:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725120 00000100:00100000:3.0:1767319080.389998:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 619 00000100:00100000:3.0:1767319080.390006:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982725120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319080.390022:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319080.390059:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982725120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (749us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319080.390066:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 619 00000100:00100000:3.0:1767319080.390109:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982725120:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319080.390222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319080.390227:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319080.855402:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319080.855457:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319080.856853:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604864 00000100:00100000:1.0:1767319080.856870:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604864 00000100:00100000:1.0:1767319080.856968:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915604736 00000100:00100000:1.0:1767319080.856973:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915604736 00000100:00100000:1.0:1767319080.856995:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 901 00000100:00100000:1.0:1767319080.857004:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604864:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319080.857107:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1182 00000100:00100000:1.0:1767319080.857109:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915604864:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (1722us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319080.857123:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915604736:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:1.0:1767319080.857125:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 901 00010000:00020000:3.0:1767319080.857162:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319080.857236:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915604736:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 105us (1767us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319080.857245:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1182 00000100:00080000:2.0:1767319080.857523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319081.093983:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319082.118881:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319083.140512:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319083.972426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319083.972442:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319083.972545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319083.972550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319083.972555:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319083.972566:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4327 00000100:00080000:0.0:1767319083.972581:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319083.972591:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319083.972621:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4e00 x1853163982725504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319083.972681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319083.972686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319083.972688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319083.972693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4327 00000100:00080000:0.0:1767319083.972696:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319083.972700:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319083.972712:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4700 x1853163982725760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319083.972720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319083.972723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319083.972725:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319083.972728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4327 00000100:00080000:0.0:1767319083.972731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319083.972735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319083.972746:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7800 x1853163982725888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319083.972769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319083.972773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319083.972774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319083.972778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4327 00000100:00080000:0.0:1767319083.972780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319083.972784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319083.972793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6300 x1853163982726272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319083.972829:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982726016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.972930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319083.972985:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982725376:0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973013:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319083.973022:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982725632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319083.973029:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319083.973058:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319083.973090:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319083.973099:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982726144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973103:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725376 00000100:00100000:0.0:1767319083.973115:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725376 00000100:00100000:2.0:1767319083.973127:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319083.973166:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726016 00000100:00100000:2.0:1767319083.973171:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726016 00000100:00100000:0.0:1767319083.973191:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725632 00000100:00100000:0.0:1767319083.973197:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725632 00000100:00100000:0.0:1767319083.973347:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902 00000100:00100000:0.0:1767319083.973359:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982725376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973427:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982725376:12345-0@lo:400:kworker.0 Request processed in 70us (399us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319083.973435:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902 00000100:00100000:0.0:1767319083.973454:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 903 00000100:00100000:0.0:1767319083.973461:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982725632:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973494:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982725632:12345-0@lo:400:kworker.0 Request processed in 34us (436us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319083.973501:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 903 00010000:00080000:0.0:1767319083.973544:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4380 x1853163982725632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319099 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319083.973559:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982725632:0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973611:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726144 00000100:00100000:0.0:1767319083.973615:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726144 00000100:00100000:0.0:1767319083.973642:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 579 00000100:00100000:0.0:1767319083.973649:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982726016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973687:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982726016:12345-0@lo:400:kworker.0 Request processed in 38us (757us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319083.973694:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 579 00000100:00100000:0.0:1767319083.973708:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 580 00000100:00100000:0.0:1767319083.973714:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982726144:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973765:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982726144:12345-0@lo:400:kworker.0 Request processed in 52us (638us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319083.973771:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 580 00000100:00080000:0.0:1767319083.973789:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319083.973794:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319083.973802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319083.973824:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982726016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319083.973859:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982726144:0@lo:400:kworker.0 00010000:00080000:0.0:1767319083.973871:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6680 x1853163982725376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319099 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319083.973880:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982725376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319083.974202:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982725504:0@lo:38:kworker.0 00000100:00100000:3.0:1767319083.974310:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319083.974351:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982725760:0@lo:38:kworker.0 00000100:00100000:3.0:1767319083.974377:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319083.974409:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982725888:0@lo:38:kworker.0 00000100:00100000:3.0:1767319083.974438:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319083.974464:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982726272:0@lo:38:kworker.0 00000100:00100000:3.0:1767319083.974485:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319083.974517:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725888 00000100:00100000:3.0:1767319083.974522:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725888 00000100:00100000:3.0:1767319083.974548:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 321 00000100:00100000:3.0:1767319083.974554:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982725888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319083.974577:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319083.974618:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982725888:12345-0@lo:38:kworker.0 Request processed in 61us (177us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319083.974625:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 321 00000100:00100000:3.0:1767319083.974659:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725504 00000100:00100000:3.0:1767319083.974663:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725504 00000100:00100000:3.0:1767319083.974693:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982725760 00000100:00100000:3.0:1767319083.974698:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982725760 00000100:00100000:2.0:1767319083.974699:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726272 00000100:00100000:3.0:1767319083.974712:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1183 00000100:00100000:2.0:1767319083.974717:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726272 00000100:00100000:3.0:1767319083.974719:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982725504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319083.974734:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319083.974770:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982725504:12345-0@lo:38:kworker.0 Request processed in 50us (501us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319083.974775:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184 00000100:00100000:2.0:1767319083.974791:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982725760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319083.974984:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319083.975075:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982725760:12345-0@lo:38:kworker.0 Request processed in 285us (695us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319083.979747:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1183 00000100:00100000:3.0:1767319083.979812:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1185 00000100:00100000:3.0:1767319083.979825:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982726272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319083.979861:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319083.979989:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982726272:12345-0@lo:38:kworker.0 Request processed in 165us (5501us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319083.979997:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1185 00000100:00080000:3.0:1767319083.980073:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319083.980079:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319083.980084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982725504:0@lo:38:kworker.0 00000100:00080000:3.0:1767319083.980099:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319083.980103:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319083.980107:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982725760:0@lo:38:kworker.0 00000100:00080000:3.0:1767319083.980119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319083.980123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319083.980126:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982725888:0@lo:38:kworker.0 00000100:00080000:3.0:1767319083.980136:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319083.980139:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319083.980142:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982726272:0@lo:38:kworker.0 00000100:00100000:2.0:1767319083.983972:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184 00010000:00010000:2.0:1767319084.165514:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767319084.943831:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915604992 offset 0 length 520: 4 00000100:00100000:0.0:1767319084.943906:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319084.944019:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605120 00000100:00100000:3.0:1767319084.944034:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605120 00000100:00100000:3.0:1767319084.944098:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1186 00000100:00100000:3.0:1767319084.944109:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915605120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319084.944145:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319084.944243:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915605120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 135us (337us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319084.944252:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1186 00000100:00100000:1.0:1767319084.957482:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319084.957959:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605248 00000100:00100000:3.0:1767319084.957975:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605248 00000100:00100000:3.0:1767319084.958144:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 904 00000100:00100000:3.0:1767319084.958158:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915605248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319084.958339:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915605248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 175us (866us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319084.958348:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 904 00000100:00080000:0.0:1767319084.962620:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319085.188920:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319085.508562:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982726400:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319085.508668:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319085.508771:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319085.508791:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982726528:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319085.508829:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319085.523960:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726400 00000100:00100000:2.0:1767319085.523983:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726400 00000100:00100000:2.0:1767319085.524071:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726528 00000100:00100000:2.0:1767319085.524076:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726528 00000100:00100000:2.0:1767319085.524101:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 620 00000100:00100000:2.0:1767319085.524111:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324e2c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982726400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319085.524156:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319085.524260:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324e2c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982726400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 142us (15584us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319085.524288:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 620 00000100:00080000:0.0:1767319085.525411:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319085.525422:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319085.525532:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982726400:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319085.526178:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 621 00000100:00100000:1.0:1767319085.526203:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982726528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319085.526245:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319085.526686:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982726528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 494us (17853us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319085.526700:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 621 00000100:00100000:1.0:1767319085.526801:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982726528:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319086.214339:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319087.237362:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319088.072007:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319063/real 1767319063] req@ffff9346333a0000 x1853163982720128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319088 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319088.072041:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346333a0000 x1853163982720128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319088 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319088.072066:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319088.072090:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319088.072094:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982720128:0@lo:250:kworker.0 00000100:00080000:3.0:1767319088.074129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319088.074156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319088.074161:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319088.074168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4312 00000100:00080000:3.0:1767319088.074172:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319088.074180:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319088.074223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cca80 x1853163982726656/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319088.074332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319088.074336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319088.074338:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319088.074342:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4332 00000100:00080000:3.0:1767319088.074344:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319088.074348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319088.074362:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf100 x1853163982726912/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319088.074391:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319088.074391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319088.074398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319088.074400:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319088.074413:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982726784:0@lo:400:kworker.0 00000100:00100000:2.0:1767319088.074480:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319088.074563:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319088.074573:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982727040:0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.074581:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726784 00000100:00100000:1.0:1767319088.074591:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726784 00000100:00100000:2.0:1767319088.074600:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319088.074654:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727040 00000100:00100000:0.0:1767319088.074655:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982726656:0@lo:250:kworker.0 00000100:00100000:1.0:1767319088.074660:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727040 00000100:00100000:2.0:1767319088.074677:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905 00000100:00100000:2.0:1767319088.074688:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982726784:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319088.074701:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4332 00000100:00080000:3.0:1767319088.074704:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319088.074708:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319088.074725:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cea00 x1853163982727168/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319088.074736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319088.074739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319088.074745:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982726784:12345-0@lo:400:kworker.0 Request processed in 57us (262us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319088.074745:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319088.074753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4332 00000100:00080000:3.0:1767319088.074756:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319088.074759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000400:00000100:0.0:1767319088.074775:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982726656 offset 0 length 520: 4 00000100:00100000:0.0:1767319088.074797:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982726912:0@lo:38:kworker.0 00000100:00100000:0.0:1767319088.074819:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319088.074840:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982727168:0@lo:38:kworker.0 00000100:00100000:0.0:1767319088.074857:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319088.074882:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982726912 00000100:00100000:0.0:1767319088.074886:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982726912 00000100:00100000:3.0:1767319088.075674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc000 x1853163982727296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319088.075721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319088.075725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319088.075728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319088.075731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4332 00000100:00080000:3.0:1767319088.075734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319088.075738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319088.075748:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059ced80 x1853163982727680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319088.075775:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727168 00000100:00100000:3.0:1767319088.075779:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727168 00000100:00100000:1.0:1767319088.077177:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 906 00000100:00100000:1.0:1767319088.077197:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982727040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.077307:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982727040:12345-0@lo:400:kworker.0 Request processed in 119us (2707us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319088.077316:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 906 00000100:00080000:1.0:1767319088.077346:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319088.077360:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319088.077373:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982727552:0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.077411:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319088.077466:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982727296:0@lo:38:kworker.0 00000100:00100000:1.0:1767319088.077493:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319088.077529:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982727680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319088.078683:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319088.078749:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727680 00000100:00100000:1.0:1767319088.078758:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727680 00000100:00100000:1.0:1767319088.078798:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187 00000100:00100000:1.0:1767319088.078805:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982727680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319088.078830:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319088.078878:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982727680:12345-0@lo:38:kworker.0 Request processed in 72us (198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319088.078885:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187 00000100:00080000:1.0:1767319088.079065:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319088.079071:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319088.079075:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982727680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319088.079132:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727552 00000100:00100000:1.0:1767319088.079136:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727552 00000100:00100000:1.0:1767319088.079189:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 581 00000100:00100000:1.0:1767319088.079196:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982727552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.079249:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982727552:12345-0@lo:400:kworker.0 Request processed in 53us (1837us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319088.079256:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 581 00000100:00080000:1.0:1767319088.079293:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319088.079309:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982727424:0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.079338:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767319088.079368:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cd180 x1853163982727040/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319104 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319088.079379:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982727040:0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.079401:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727424 00000100:00100000:1.0:1767319088.079404:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727424 00000100:00100000:1.0:1767319088.079437:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 582 00000100:00100000:1.0:1767319088.079443:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982727424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.079475:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982727424:12345-0@lo:400:kworker.0 Request processed in 32us (135us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319088.079483:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 582 00000100:00080000:1.0:1767319088.079495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319088.079509:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982727424:0@lo:400:kworker.0 00010000:00080000:1.0:1767319088.079534:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cc380 x1853163982726784/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319104 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319088.079740:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905 00000100:00100000:2.0:1767319088.079785:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727296 00000100:00100000:2.0:1767319088.079791:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727296 00000100:00100000:2.0:1767319088.079813:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 322 00000100:00100000:2.0:1767319088.079819:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982727296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319088.079834:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319088.079873:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982727296:12345-0@lo:38:kworker.0 Request processed in 53us (2380us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319088.079880:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 322 00000100:00080000:2.0:1767319088.079908:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319088.079912:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319088.079915:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982727296:0@lo:38:kworker.0 00000100:00100000:1.0:1767319088.080650:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982726784:0@lo:400:kworker.0 00000100:00100000:1.0:1767319088.080701:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982727552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319088.081805:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188 00000100:00100000:0.0:1767319088.081815:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982726912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319088.081838:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319088.081916:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982726912:12345-0@lo:38:kworker.0 Request processed in 101us (7095us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319088.081923:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188 00000100:00100000:3.0:1767319088.082162:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1189 00000100:00100000:3.0:1767319088.082168:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982727168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319088.082180:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319088.082229:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982727168:12345-0@lo:38:kworker.0 Request processed in 59us (7370us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319088.082235:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1189 00000100:00080000:1.0:1767319088.082334:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319088.082340:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319088.082345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982727168:0@lo:38:kworker.0 00000100:00080000:1.0:1767319088.082358:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319088.082362:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319088.082365:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982726912:0@lo:38:kworker.0 00010000:00010000:1.0:1767319088.260505:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319089.284523:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319090.064474:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319090.064764:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605504 00000100:00100000:3.0:1767319090.064775:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605504 00000100:00100000:3.0:1767319090.064849:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 907 00000100:00100000:3.0:1767319090.064860:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915605504:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767319090.064865:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319090.064936:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915605504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (471us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319090.064943:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 907 00000100:00100000:2.0:1767319090.076556:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319090.076682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605376 00000100:00100000:3.0:1767319090.076697:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605376 00000100:00100000:3.0:1767319090.076757:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1190 00000100:00100000:3.0:1767319090.076770:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915605376:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319090.076804:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319090.076897:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915605376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 128us (359us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319090.076904:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1190 00010000:00010000:2.0:1767319090.308567:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319090.631372:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319090.631423:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982727808:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319090.631546:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319090.632307:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727808 00000100:00100000:1.0:1767319090.632321:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727808 00000100:00100000:1.0:1767319090.632422:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 622 00000100:00100000:1.0:1767319090.632433:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982727808:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319090.632476:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319090.632556:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982727808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 124us (1010us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319090.632564:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 622 00000100:00100000:1.0:1767319090.632630:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982727808:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319090.632954:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319090.633305:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982727936:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319090.633358:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319090.633432:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982727936 00000100:00100000:1.0:1767319090.633439:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982727936 00000100:00100000:1.0:1767319090.633489:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 623 00000100:00100000:1.0:1767319090.633508:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982727936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319090.633530:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767319090.633590:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319090.640053:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982727936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6541us (6678us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319090.640073:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 623 00000100:00100000:2.0:1767319090.640414:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982727936:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319091.334856:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319092.356692:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319093.193012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319093.193024:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319093.193151:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767319093.193176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319093.193180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319093.193183:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319093.193191:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4336 00000100:00080000:0.0:1767319093.193196:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319093.193203:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319093.193229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5500 x1853163982728192/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319093.193358:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982728192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319093.193478:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319093.193581:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982728064:0@lo:400:kworker.0 00000100:00100000:1.0:1767319093.193588:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728192 00000100:00100000:1.0:1767319093.193600:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728192 00000100:00100000:1.0:1767319093.193670:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191 00000100:00100000:2.0:1767319093.193670:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319093.193688:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728192:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319093.193726:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319093.193784:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728192:12345-0@lo:38:kworker.0 Request processed in 98us (307us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319093.193793:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191 00000100:00080000:0.0:1767319093.193835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319093.193840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319093.193843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319093.193847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4336 00000100:00080000:0.0:1767319093.193850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319093.193855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319093.193873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5880 x1853163982728448/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319093.193883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319093.193887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319093.193888:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319093.193891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4336 00000100:00080000:0.0:1767319093.193893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319093.193896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319093.193906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6300 x1853163982728576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319093.193930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319093.193933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319093.193935:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319093.193938:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4336 00000100:00080000:0.0:1767319093.193940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319093.193943:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319093.193952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4e00 x1853163982728960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319093.193979:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982728704:0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.194013:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319093.197901:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319093.197923:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982728320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.198010:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319093.198018:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982728832:0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.198049:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319093.198146:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982728448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319093.198193:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728704 00000100:00100000:0.0:1767319093.198202:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728704 00000100:00100000:0.0:1767319093.198301:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728832 00000100:00100000:0.0:1767319093.198305:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728832 00000100:00100000:0.0:1767319093.198330:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 583 00000100:00100000:0.0:1767319093.198338:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982728704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.198398:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982728704:12345-0@lo:400:kworker.0 Request processed in 59us (4382us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319093.198406:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 583 00000100:00100000:0.0:1767319093.198423:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 584 00000100:00100000:0.0:1767319093.198429:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982728832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.198463:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982728832:12345-0@lo:400:kworker.0 Request processed in 35us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319093.198468:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 584 00000100:00100000:0.0:1767319093.198506:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982728832:0@lo:400:kworker.0 00000100:00080000:0.0:1767319093.198556:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319093.198561:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319093.200061:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319093.200649:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728064 00000100:00100000:1.0:1767319093.200667:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728064 00000100:00100000:1.0:1767319093.200764:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908 00000100:00100000:1.0:1767319093.200778:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982728064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319093.200910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982728064:12345-0@lo:400:kworker.0 Request processed in 135us (7251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319093.200919:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908 00000100:00080000:1.0:1767319093.200956:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319093.201004:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6d80 x1853163982728064/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319109 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319093.201021:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982728064:0@lo:400:kworker.0 00000100:00100000:2.0:1767319093.204127:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982728704:0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.204410:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728320 00000100:00100000:0.0:1767319093.204419:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728320 00000100:00100000:0.0:1767319093.204481:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909 00000100:00100000:0.0:1767319093.204490:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982728320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319093.204565:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982728320:12345-0@lo:400:kworker.0 Request processed in 74us (4507us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319093.204572:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909 00000100:00080000:2.0:1767319093.204641:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319093.204666:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5180 x1853163982728320/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319109 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319093.204676:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982728320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319093.204774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319093.204969:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982728576:0@lo:38:kworker.0 00000100:00100000:1.0:1767319093.205008:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319093.205037:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982728960:0@lo:38:kworker.0 00000100:00100000:1.0:1767319093.205059:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319093.205087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319093.205092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319093.205096:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982728192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319093.205136:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728448 00000100:00100000:1.0:1767319093.205140:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728448 00000100:00100000:1.0:1767319093.205162:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728960 00000100:00100000:1.0:1767319093.205165:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728960 00000100:00100000:1.0:1767319093.205176:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1192 00000100:00100000:1.0:1767319093.205181:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319093.205200:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319093.205235:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728448:12345-0@lo:38:kworker.0 Request processed in 52us (460us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319093.205241:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1192 00000100:00100000:1.0:1767319093.205255:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193 00000100:00100000:1.0:1767319093.205258:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728960:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319093.205385:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982728576 00000100:00100000:2.0:1767319093.205390:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982728576 00000100:00100000:2.0:1767319093.205413:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 323 00000100:00100000:2.0:1767319093.205418:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982728576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319093.205429:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319093.205458:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982728576:12345-0@lo:38:kworker.0 Request processed in 39us (450us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319093.205465:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 323 00000100:00080000:0.0:1767319093.205577:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319093.205582:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319093.205585:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982728448:0@lo:38:kworker.0 00000100:00080000:0.0:1767319093.205594:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319093.205597:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319093.205600:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982728576:0@lo:38:kworker.0 00010000:00020000:1.0:1767319093.206417:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319093.206471:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982728960:12345-0@lo:38:kworker.0 Request processed in 1210us (1410us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319093.206478:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193 00000100:00080000:0.0:1767319093.206642:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319093.206647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319093.206651:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982728960:0@lo:38:kworker.0 00010000:00010000:2.0:1767319093.380629:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319094.404984:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319095.180011:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319095.180392:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605760 00000100:00100000:0.0:1767319095.180405:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605760 00000100:00080000:2.0:1767319095.180548:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319095.180655:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 910 00000100:00100000:0.0:1767319095.180668:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915605760:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319095.180925:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915605760:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 260us (927us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319095.180934:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 910 00000100:00100000:3.0:1767319095.181097:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319095.208454:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605632 00000100:00100000:1.0:1767319095.208469:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605632 00000100:00100000:1.0:1767319095.208550:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1194 00000100:00100000:1.0:1767319095.208560:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915605632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319095.208585:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319095.208671:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915605632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 110us (27573us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319095.208678:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1194 00010000:00010000:1.0:1767319095.428588:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319095.748629:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982729088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319095.748709:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319095.748898:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729088 00000100:00100000:1.0:1767319095.748908:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729088 00000100:00100000:1.0:1767319095.748978:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 624 00000100:00100000:1.0:1767319095.748989:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982729088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319095.749019:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319095.749092:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982729088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (383us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319095.749104:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 624 00000100:00080000:0.0:1767319095.749119:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319095.749156:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982729088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319095.749171:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319095.749181:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982729216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319095.749209:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319095.749287:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729216 00000100:00100000:1.0:1767319095.749292:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729216 00000100:00100000:1.0:1767319095.749331:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 625 00000100:00100000:1.0:1767319095.749338:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982729216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319095.749355:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319095.749396:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982729216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (187us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319095.749403:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 625 00000100:00080000:0.0:1767319095.749453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319095.749656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982729216:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319096.452588:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319097.476526:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319098.308576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319098.308588:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319098.308692:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319098.308747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319098.308752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319098.308756:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319098.308764:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4341 00000100:00080000:3.0:1767319098.308769:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319098.308776:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319098.308802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc380 x1853163982729472/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319098.308850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319098.308853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319098.308855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319098.308859:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4341 00000100:00080000:3.0:1767319098.308861:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319098.308864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319098.308876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cd180 x1853163982729728/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319098.308884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319098.308887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319098.308889:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319098.308891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4341 00000100:00080000:3.0:1767319098.308893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319098.308896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319098.308906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059ced80 x1853163982729856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319098.308964:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982729600:0@lo:400:kworker.0 00000100:00100000:2.0:1767319098.309041:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319098.309077:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319098.309083:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982729984:0@lo:400:kworker.0 00000100:00100000:2.0:1767319098.309125:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319098.309157:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729600 00000100:00100000:2.0:1767319098.309165:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729600 00000100:00100000:2.0:1767319098.309217:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 911 00000100:00100000:2.0:1767319098.309227:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982729600:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319098.309302:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982729600:12345-0@lo:400:kworker.0 Request processed in 76us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319098.309310:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 911 00010000:00080000:2.0:1767319098.309350:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cc700 x1853163982729600/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319114 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319098.309364:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982729600:0@lo:400:kworker.0 00000100:00100000:0.0:1767319098.310260:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729984 00000100:00100000:0.0:1767319098.310268:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729984 00000100:00100000:0.0:1767319098.310346:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 585 00000100:00100000:0.0:1767319098.310355:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982729984:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319098.310413:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982729984:12345-0@lo:400:kworker.0 Request processed in 60us (1287us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319098.310420:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 585 00000100:00080000:0.0:1767319098.310440:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319098.310444:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319098.310458:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982729472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319098.310491:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319098.310511:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982729728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319098.310532:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319098.310553:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982729856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319098.310575:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319098.310618:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729472 00000100:00100000:0.0:1767319098.310622:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729472 00000100:00100000:0.0:1767319098.310643:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729728 00000100:00100000:0.0:1767319098.310646:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729728 00000100:00100000:0.0:1767319098.310660:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1195 00000100:00100000:0.0:1767319098.310665:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982729472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319098.310685:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319098.310717:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982729472:12345-0@lo:38:kworker.0 Request processed in 51us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319098.310723:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1195 00000100:00080000:0.0:1767319098.310747:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319098.310752:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319098.310755:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982729472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319098.310775:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196 00000100:00100000:0.0:1767319098.310779:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982729728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319098.310789:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319098.310818:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982729728:12345-0@lo:38:kworker.0 Request processed in 36us (284us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319098.310824:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196 00000100:00080000:0.0:1767319098.310843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319098.310847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319098.310850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982729728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319098.310956:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729856 00000100:00100000:2.0:1767319098.310960:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729856 00000100:00100000:2.0:1767319098.310990:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 324 00000100:00100000:2.0:1767319098.310995:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982729856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319098.311005:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319098.311051:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982729856:12345-0@lo:38:kworker.0 Request processed in 54us (474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319098.311057:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 324 00000100:00100000:2.0:1767319098.311094:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982729984:0@lo:400:kworker.0 00000100:00080000:2.0:1767319098.311131:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319098.311135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319098.311138:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982729856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319098.311367:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982730112:0@lo:400:kworker.0 00000100:00100000:1.0:1767319098.311402:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319098.311431:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982729344:0@lo:400:kworker.0 00000100:00100000:1.0:1767319098.311451:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319098.311525:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730112 00000100:00100000:2.0:1767319098.311530:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730112 00000100:00100000:2.0:1767319098.311571:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 586 00000100:00100000:2.0:1767319098.311578:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982730112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319098.311616:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982730112:12345-0@lo:400:kworker.0 Request processed in 38us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319098.311622:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 586 00000100:00100000:0.0:1767319098.311686:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982729344 00000100:00100000:0.0:1767319098.311690:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982729344 00000100:00100000:0.0:1767319098.311723:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912 00000100:00100000:0.0:1767319098.311729:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982729344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319098.311762:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982729344:12345-0@lo:400:kworker.0 Request processed in 32us (309us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319098.311768:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912 00000100:00080000:1.0:1767319098.311851:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319098.311855:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319098.311876:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982730112:0@lo:400:kworker.0 00010000:00080000:1.0:1767319098.311888:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cf800 x1853163982729344/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319114 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319098.311897:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982729344:0@lo:400:kworker.0 00000100:00080000:3.0:1767319098.317515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319098.317526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319098.317530:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319098.317538:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4341 00000100:00080000:3.0:1767319098.317544:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319098.317553:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319098.317594:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163982730240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319098.318084:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982730240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319098.318163:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319098.318331:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730240 00000100:00100000:2.0:1767319098.318339:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730240 00000100:00100000:2.0:1767319098.318392:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197 00000100:00100000:2.0:1767319098.318399:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982730240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319098.318419:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319098.318474:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982730240:12345-0@lo:38:kworker.0 Request processed in 74us (310us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319098.318481:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197 00000100:00080000:3.0:1767319098.318561:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319098.318566:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319098.318570:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982730240:0@lo:38:kworker.0 00010000:00010000:1.0:1767319098.500768:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319099.525160:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319100.300233:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319100.300357:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915605888 00000100:00100000:0.0:1767319100.300369:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915605888 00000100:00100000:0.0:1767319100.300422:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1198 00000100:00100000:0.0:1767319100.300779:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915605888:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319100.300843:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319100.300912:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915605888:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 137us (691us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319100.300920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1198 00000100:00100000:2.0:1767319100.311126:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319100.311301:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606016 00000100:00100000:0.0:1767319100.311312:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606016 00000100:00100000:0.0:1767319100.311394:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 913 00000100:00100000:0.0:1767319100.311405:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915606016:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767319100.311598:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319100.311694:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915606016:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 290us (577us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319100.311702:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 913 00010000:00010000:1.0:1767319100.548542:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319100.869609:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319100.869632:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982730368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319100.869696:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319100.869778:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730368 00000100:00100000:3.0:1767319100.869787:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730368 00000100:00100000:3.0:1767319100.869837:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 626 00000100:00100000:3.0:1767319100.869845:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982730368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319100.869874:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319100.869912:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982730368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319100.869919:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 626 00000100:00080000:3.0:1767319100.869942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319100.869985:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982730368:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319100.874400:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982730496:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319100.874483:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319100.875200:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730496 00000100:00100000:0.0:1767319100.875210:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730496 00000100:00100000:0.0:1767319100.875266:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 627 00000100:00100000:0.0:1767319100.875298:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982730496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319100.875326:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319100.875373:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982730496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (890us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319100.875395:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 627 00000100:00080000:0.0:1767319100.875415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319100.876380:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982730496:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319101.572604:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319102.597291:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319103.428651:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319103.428661:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319103.428726:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319103.428733:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319103.428736:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319103.428744:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4346 00000100:00080000:1.0:1767319103.428749:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319103.428755:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319103.428778:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582a00 x1853163982730752/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319103.428828:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319103.428831:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319103.428833:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319103.428837:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4346 00000100:00080000:1.0:1767319103.428840:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319103.428845:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319103.428856:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580a80 x1853163982731008/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319103.428868:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319103.428871:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319103.428874:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319103.428875:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982730752:0@lo:38:kworker.0 00000100:00080000:1.0:1767319103.428879:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4346 00000100:00080000:1.0:1767319103.428884:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319103.428889:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319103.428901:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581c00 x1853163982731136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319103.428934:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319103.428938:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319103.428940:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319103.428943:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4346 00000100:00080000:1.0:1767319103.428946:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319103.428949:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319103.428960:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982731520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319103.428973:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319103.428986:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982730880:0@lo:400:kworker.0 00000100:00100000:1.0:1767319103.429024:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319103.429044:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982731008:0@lo:38:kworker.0 00000100:00100000:1.0:1767319103.429052:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319103.429061:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982731264:0@lo:400:kworker.0 00000100:00100000:0.0:1767319103.429087:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319103.429095:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319103.429133:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982731136:0@lo:38:kworker.0 00000100:00100000:1.0:1767319103.429151:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730752 00000100:00100000:0.0:1767319103.429160:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319103.429160:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730752 00000100:00100000:0.0:1767319103.429188:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982731520:0@lo:38:kworker.0 00000100:00100000:1.0:1767319103.429201:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730880 00000100:00100000:1.0:1767319103.429206:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730880 00000100:00100000:0.0:1767319103.429214:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319103.429248:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731008 00000100:00100000:0.0:1767319103.429251:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731520 00000100:00100000:1.0:1767319103.429252:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731008 00000100:00100000:0.0:1767319103.429257:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731520 00000100:00100000:0.0:1767319103.429314:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 914 00000100:00100000:0.0:1767319103.429330:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982730880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319103.429381:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982730880:12345-0@lo:400:kworker.0 Request processed in 59us (356us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319103.429443:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982731392:0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.429472:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319103.429494:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319103.429499:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982730624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.429520:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319103.429540:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731264 00000100:00100000:2.0:1767319103.429544:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731264 00000100:00100000:2.0:1767319103.429573:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731392 00000100:00100000:2.0:1767319103.429576:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731392 00000100:00100000:2.0:1767319103.429596:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 587 00000100:00100000:2.0:1767319103.429603:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982731264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.429635:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982731264:12345-0@lo:400:kworker.0 Request processed in 32us (541us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319103.429641:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 587 00000100:00100000:2.0:1767319103.429657:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 588 00000100:00100000:2.0:1767319103.429662:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982731392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.429692:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982731392:12345-0@lo:400:kworker.0 Request processed in 30us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319103.429698:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 588 00000100:00100000:2.0:1767319103.429738:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982731392:0@lo:400:kworker.0 00000100:00080000:2.0:1767319103.429774:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319103.429778:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319103.429781:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319103.429794:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731136 00000100:00100000:2.0:1767319103.429798:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731136 00000100:00100000:2.0:1767319103.429821:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 325 00000100:00100000:2.0:1767319103.429826:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982731136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319103.429846:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319103.429883:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982731136:12345-0@lo:38:kworker.0 Request processed in 55us (722us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319103.429889:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 325 00000100:00100000:2.0:1767319103.429931:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982731264:0@lo:400:kworker.0 00010000:00080000:2.0:1767319103.429942:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583800 x1853163982730880/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319119 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319103.429951:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982730880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.429970:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982730624 00000100:00100000:2.0:1767319103.429972:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982730624 00000100:00080000:2.0:1767319103.429986:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319103.430097:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767319103.430103:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982730752:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319103.430113:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319103.430143:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982730752:12345-0@lo:38:kworker.0 Request processed in 40us (1173us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319103.430149:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767319103.430164:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200 00000100:00100000:1.0:1767319103.430168:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982731520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319103.430179:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319103.430204:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982731520:12345-0@lo:38:kworker.0 Request processed in 35us (991us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319103.430210:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200 00000100:00100000:1.0:1767319103.430223:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1201 00000100:00100000:1.0:1767319103.430226:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982731008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319103.430236:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319103.430259:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982731008:12345-0@lo:38:kworker.0 Request processed in 31us (1173us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319103.430264:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1201 00000100:00080000:1.0:1767319103.430331:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319103.430336:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319103.430967:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 914 00000100:00100000:2.0:1767319103.431415:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915 00000100:00100000:2.0:1767319103.431430:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982730624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319103.431497:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982730624:12345-0@lo:400:kworker.0 Request processed in 76us (1976us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319103.431504:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915 00010000:00080000:2.0:1767319103.431533:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581500 x1853163982730624/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319119 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319103.431552:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982730624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319103.431841:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982731008:0@lo:38:kworker.0 00000100:00080000:1.0:1767319103.431862:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319103.431866:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319103.431869:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982731136:0@lo:38:kworker.0 00000100:00080000:1.0:1767319103.431878:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319103.431880:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319103.431883:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982731520:0@lo:38:kworker.0 00000100:00080000:1.0:1767319103.431891:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319103.431895:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319103.431897:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982730752:0@lo:38:kworker.0 00010000:00010000:1.0:1767319103.622212:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319104.644636:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319105.426027:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319105.426215:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606144 00000100:00100000:0.0:1767319105.426228:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606144 00000100:00100000:0.0:1767319105.426346:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1202 00000100:00100000:0.0:1767319105.426356:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915606144:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319105.426382:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319105.426480:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915606144:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 123us (463us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319105.426488:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1202 00000100:00100000:1.0:1767319105.427569:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319105.428146:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606272 00000100:00100000:0.0:1767319105.428158:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606272 00000100:00100000:0.0:1767319105.428250:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 916 00000100:00100000:0.0:1767319105.428260:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915606272:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319105.428385:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915606272:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 126us (821us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319105.428395:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 916 00000100:00080000:3.0:1767319105.429079:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319105.670719:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319105.988605:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319105.988635:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982731648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319105.988800:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319105.989611:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982731776:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319105.989653:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319105.990344:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731648 00000100:00100000:0.0:1767319105.990353:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731648 00000100:00100000:0.0:1767319105.990412:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731776 00000100:00100000:0.0:1767319105.990416:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731776 00000100:00100000:0.0:1767319105.990435:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 628 00000100:00100000:0.0:1767319105.990443:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982731648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319105.990473:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319105.990529:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982731648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (1729us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319105.990536:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 628 00000100:00100000:0.0:1767319105.990553:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 629 00000100:00100000:0.0:1767319105.990560:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982731776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319105.990580:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319105.990611:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982731776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (957us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319105.990617:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 629 00000100:00100000:0.0:1767319105.990662:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982731776:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319105.990691:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982731648:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319105.991040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319105.991044:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319106.696097:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319107.716506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319108.548794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319108.548805:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319108.548880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319108.548884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319108.548888:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319108.548896:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4351 00000100:00080000:0.0:1767319108.548901:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319108.548908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319108.548918:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982731904:0@lo:400:kworker.0 00000100:00100000:0.0:1767319108.548929:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982732032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319108.548974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319108.548978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319108.548979:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319108.548982:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4351 00000100:00080000:0.0:1767319108.548985:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319108.548989:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319108.548989:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319108.549000:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4e00 x1853163982732288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319108.549008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319108.549011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319108.549012:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319108.549015:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4351 00000100:00080000:0.0:1767319108.549017:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319108.549020:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319108.549033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015180 x1853163982732416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319108.549064:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982731904 00000100:00100000:3.0:1767319108.549072:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982731904 00000100:00100000:2.0:1767319108.549101:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982732032:0@lo:38:kworker.0 00000100:00100000:3.0:1767319108.549125:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917 00000100:00100000:2.0:1767319108.549129:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319108.549148:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982731904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319108.549154:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982732288:0@lo:38:kworker.0 00000100:00100000:2.0:1767319108.549171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319108.549190:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982732416:0@lo:38:kworker.0 00000100:00100000:1.0:1767319108.549230:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732032 00000100:00100000:1.0:1767319108.549234:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732032 00000100:00100000:1.0:1767319108.549253:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732288 00000100:00100000:1.0:1767319108.549256:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732288 00000100:00100000:1.0:1767319108.549267:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203 00000100:00100000:1.0:1767319108.549305:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982732032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319108.549323:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319108.549357:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982732032:12345-0@lo:38:kworker.0 Request processed in 52us (226us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319108.549364:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203 00000100:00100000:1.0:1767319108.549382:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1204 00000100:00100000:1.0:1767319108.549386:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982732288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319108.549394:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319108.549417:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982732288:12345-0@lo:38:kworker.0 Request processed in 29us (244us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319108.549422:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1204 00000100:00100000:1.0:1767319108.549443:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982732672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.549468:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319108.549495:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732672 00000100:00100000:1.0:1767319108.549498:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732672 00000100:00100000:1.0:1767319108.550020:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 589 00000100:00100000:1.0:1767319108.550036:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982732672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.550083:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982732672:12345-0@lo:400:kworker.0 Request processed in 55us (613us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319108.550090:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 589 00000100:00100000:1.0:1767319108.550133:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982732672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.550174:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982732160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.550201:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319108.550221:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319108.550227:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982732544:0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.550248:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319108.550401:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319108.550462:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732160 00000100:00100000:1.0:1767319108.550467:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732160 00000100:00080000:0.0:1767319108.550554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319108.550559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319108.550561:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319108.550565:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4351 00000100:00080000:0.0:1767319108.550569:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319108.550574:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319108.550596:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163982732800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319108.558974:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982732800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319108.559055:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319108.559103:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319108.559107:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319108.559111:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982732032:0@lo:38:kworker.0 00000100:00080000:2.0:1767319108.559122:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319108.559125:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319108.559128:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982732288:0@lo:38:kworker.0 00000100:00100000:1.0:1767319108.559910:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732800 00000100:00100000:1.0:1767319108.559922:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732800 00000100:00100000:1.0:1767319108.559975:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 918 00000100:00100000:1.0:1767319108.559990:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982732160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.560073:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982732160:12345-0@lo:400:kworker.0 Request processed in 89us (9870us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319108.560081:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 918 00000100:00100000:1.0:1767319108.560101:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205 00000100:00100000:1.0:1767319108.560105:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982732800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319108.560124:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319108.560396:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982732800:12345-0@lo:38:kworker.0 Request processed in 287us (1339us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319108.560404:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205 00000100:00100000:1.0:1767319108.560437:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732544 00000100:00100000:1.0:1767319108.560442:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732544 00000100:00100000:1.0:1767319108.560485:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 590 00000100:00100000:1.0:1767319108.560491:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982732544:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319108.560526:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982732544:12345-0@lo:400:kworker.0 Request processed in 34us (10276us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319108.560531:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 590 00000100:00100000:1.0:1767319108.560583:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982732544:0@lo:400:kworker.0 00010000:00080000:1.0:1767319108.560597:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4700 x1853163982732160/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319124 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319108.560607:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982732160:0@lo:400:kworker.0 00000100:00080000:1.0:1767319108.560692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319108.560696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319108.560699:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319108.560777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319108.560783:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319108.561228:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982732800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319108.561260:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732416 00000100:00100000:2.0:1767319108.561367:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732416 00000100:00100000:2.0:1767319108.561399:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 326 00000100:00100000:2.0:1767319108.561404:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982732416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319108.561419:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319108.561462:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982732416:12345-0@lo:38:kworker.0 Request processed in 56us (11061us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319108.561468:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 326 00000100:00080000:2.0:1767319108.561494:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319108.561499:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319108.561502:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982732416:0@lo:38:kworker.0 00000100:00100000:3.0:1767319108.562394:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982731904:12345-0@lo:400:kworker.0 Request processed in 13256us (13400us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319108.562405:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917 00010000:00080000:1.0:1767319108.562538:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7800 x1853163982731904/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319124 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319108.562552:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982731904:0@lo:400:kworker.0 00010000:00010000:2.0:1767319108.744755:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319109.765242:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319110.540752:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319110.543007:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606656 00000100:00100000:3.0:1767319110.543022:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606656 00000100:00100000:3.0:1767319110.543121:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 919 00000100:00100000:3.0:1767319110.543132:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915606656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319110.543362:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319110.543920:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915606656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 787us (3178us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319110.543930:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 919 00000400:00000100:1.0:1767319110.544083:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915606400 offset 0 length 520: 4 00000100:00100000:1.0:1767319110.544207:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319110.544450:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606528 00000100:00100000:3.0:1767319110.544469:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606528 00000100:00100000:3.0:1767319110.544504:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1206 00000100:00100000:3.0:1767319110.544510:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915606528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319110.544530:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319110.544601:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915606528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 89us (394us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319110.544609:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1206 00010000:00010000:1.0:1767319110.788620:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319111.112137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982732928:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319111.112218:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319111.112328:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982732928 00000100:00100000:1.0:1767319111.112336:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982732928 00000100:00100000:1.0:1767319111.112394:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 630 00000100:00100000:1.0:1767319111.112403:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982732928:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319111.112437:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319111.112484:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982732928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319111.112492:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 630 00000100:00100000:1.0:1767319111.112546:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982732928:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319111.113208:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319111.113262:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319111.113299:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982733056:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319111.113341:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319111.113712:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733056 00000100:00100000:2.0:1767319111.113720:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733056 00000100:00100000:2.0:1767319111.113808:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 631 00000100:00100000:2.0:1767319111.113818:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982733056:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319111.113842:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319111.113884:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982733056:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (542us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319111.113891:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 631 00000100:00080000:3.0:1767319111.114174:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319111.114209:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982733056:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319111.814686:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319112.836528:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319113.668485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319113.668715:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319088/real 1767319088] req@ffff9346059cca80 x1853163982726656/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319113 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319113.668852:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346059cca80 x1853163982726656/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319113 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319113.668889:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319113.668959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319113.668964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319113.668967:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319113.668973:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4357 00000100:00080000:2.0:1767319113.668976:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319113.668983:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319113.669002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377480 x1853163982733312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319113.669029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319113.669032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319113.669034:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319113.669037:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4357 00000100:00080000:2.0:1767319113.669039:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319113.669043:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319113.669054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604376680 x1853163982733568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319113.669062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319113.669065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319113.669066:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319113.669069:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4357 00000100:00080000:2.0:1767319113.669071:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319113.669074:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319113.669084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375f80 x1853163982733696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319113.669112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319113.669115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319113.669117:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319113.669120:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4357 00000100:00080000:2.0:1767319113.669123:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319113.669126:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319113.669136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163982734080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319113.669230:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982733952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669335:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319113.669395:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319113.669402:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982733184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669452:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319113.669488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982733824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669511:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319113.669530:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319113.669535:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982733440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669557:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319113.669583:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733952 00000100:00100000:1.0:1767319113.669591:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733952 00000100:00100000:1.0:1767319113.669638:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733824 00000100:00100000:1.0:1767319113.669641:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733824 00000100:00100000:1.0:1767319113.669662:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 591 00000100:00100000:1.0:1767319113.669675:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982733952:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319113.669708:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319113.669713:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319113.669735:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982733952:12345-0@lo:400:kworker.0 Request processed in 55us (390us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319113.669743:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 591 00000100:00100000:3.0:1767319113.669747:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733184 00000100:00100000:3.0:1767319113.669751:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733184 00000100:00100000:1.0:1767319113.669760:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 592 00000100:00100000:1.0:1767319113.669766:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982733824:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319113.669777:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319113.669796:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982733824:12345-0@lo:400:kworker.0 Request processed in 30us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319113.669802:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 592 00000100:00100000:3.0:1767319113.669802:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733440 00000100:00100000:3.0:1767319113.669804:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733440 00000100:00080000:3.0:1767319113.669815:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319113.669832:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767319113.669835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982733824:0@lo:400:kworker.0 00000100:00100000:3.0:1767319113.669838:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982733184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669858:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921 00000100:00100000:1.0:1767319113.669864:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982733440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319113.669870:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982733184:12345-0@lo:400:kworker.0 Request processed in 33us (418us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319113.669875:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767319113.669891:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982733440:12345-0@lo:400:kworker.0 Request processed in 28us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319113.669897:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921 00010000:00080000:1.0:1767319113.669920:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375500 x1853163982733440/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319129 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319113.669929:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982733440:0@lo:400:kworker.0 00010000:00080000:1.0:1767319113.669950:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375880 x1853163982733184/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319129 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319113.669957:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982733184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319113.669965:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982733952:0@lo:400:kworker.0 00000100:00080000:0.0:1767319113.676327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319113.676440:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319113.676449:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982726656:0@lo:250:kworker.0 00000100:00100000:0.0:1767319113.676490:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982733312:0@lo:38:kworker.0 00000100:00100000:0.0:1767319113.676579:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319113.676645:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982733568:0@lo:38:kworker.0 00000100:00100000:0.0:1767319113.676669:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319113.676702:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982733696:0@lo:38:kworker.0 00000100:00100000:0.0:1767319113.676739:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319113.676785:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982734080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319113.676813:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319113.676872:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733696 00000100:00100000:0.0:1767319113.676880:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733696 00000100:00100000:0.0:1767319113.676918:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 327 00000100:00100000:0.0:1767319113.676930:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982733696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319113.676950:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319113.676992:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982733696:12345-0@lo:38:kworker.0 Request processed in 61us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319113.677000:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 327 00000100:00080000:0.0:1767319113.677040:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319113.677044:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319113.677047:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982733696:0@lo:38:kworker.0 00000100:00100000:3.0:1767319113.677353:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733312 00000100:00100000:3.0:1767319113.677359:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733312 00000100:00100000:3.0:1767319113.677382:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982733568 00000100:00100000:3.0:1767319113.677385:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982733568 00000100:00100000:3.0:1767319113.677401:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734080 00000100:00100000:3.0:1767319113.677404:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734080 00000100:00100000:3.0:1767319113.677415:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1207 00000100:00100000:3.0:1767319113.677420:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982733312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319113.677433:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319113.677465:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208 00000100:00100000:0.0:1767319113.677472:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982733568:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319113.677479:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982733312:12345-0@lo:38:kworker.0 Request processed in 56us (897us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767319113.677486:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319113.677491:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1207 00000100:00100000:3.0:1767319113.677521:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209 00000100:00100000:0.0:1767319113.677526:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982733568:12345-0@lo:38:kworker.0 Request processed in 51us (854us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319113.677531:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982734080:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319113.677538:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208 00010000:00020000:3.0:1767319113.677545:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319113.677570:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319113.677576:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319113.677581:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982734080:12345-0@lo:38:kworker.0 Request processed in 48us (764us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319113.677583:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982733312:0@lo:38:kworker.0 00000100:00080000:0.0:1767319113.677597:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319113.677601:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319113.677606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982733568:0@lo:38:kworker.0 00000100:00100000:3.0:1767319113.677610:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209 00000100:00080000:0.0:1767319113.677626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319113.677631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319113.677634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982734080:0@lo:38:kworker.0 00000100:00080000:2.0:1767319113.677705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319113.677710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319113.677712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319113.677717:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4336 00000100:00080000:2.0:1767319113.677720:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319113.677735:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319113.677846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa300 x1853163982734208/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319113.677902:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982734208:0@lo:250:kworker.0 00000400:00000100:0.0:1767319113.677928:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982734208 offset 0 length 520: 4 00010000:00010000:1.0:1767319113.861809:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319114.884514:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319115.665214:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319115.665301:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606784 00000100:00100000:1.0:1767319115.665311:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606784 00000100:00100000:1.0:1767319115.665357:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1210 00000100:00100000:1.0:1767319115.665364:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915606784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319115.665386:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319115.665449:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915606784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 84us (245us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319115.665464:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1210 00000100:00100000:2.0:1767319115.668310:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319115.668394:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915606912 00000100:00100000:3.0:1767319115.668404:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915606912 00000100:00100000:3.0:1767319115.668468:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 922 00000100:00080000:0.0:1767319115.668475:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319115.668478:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915606912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319115.668560:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915606912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (291us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319115.668569:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 922 00010000:00010000:1.0:1767319115.908482:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319116.230297:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982734336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319116.230369:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319116.230509:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734336 00000100:00100000:0.0:1767319116.230517:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734336 00000100:00100000:2.0:1767319116.230700:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982734464:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319116.230730:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319116.230769:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319116.236355:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734464 00000100:00100000:0.0:1767319116.236370:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734464 00000100:00100000:0.0:1767319116.236451:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 632 00000100:00100000:0.0:1767319116.236464:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982734336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319116.236498:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319116.236569:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982734336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 109us (6199us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319116.236576:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 632 00000100:00100000:0.0:1767319116.236594:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 633 00000100:00100000:0.0:1767319116.236601:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982734464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319116.236616:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319116.236647:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982734464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (5917us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319116.236653:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 633 00000100:00080000:2.0:1767319116.237307:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319116.237357:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982734464:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319116.237422:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982734336:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319116.933723:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319117.957958:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319118.788541:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319118.788551:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319118.788638:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319118.788642:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319118.788647:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319118.788656:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4362 00000100:00080000:3.0:1767319118.788661:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319118.788668:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319118.788694:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163982734720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319118.788740:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319118.788744:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319118.788746:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319118.788751:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4362 00000100:00080000:3.0:1767319118.788753:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319118.788758:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319118.788764:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982734720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319118.788777:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982734976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319118.788788:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319118.788791:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319118.788793:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319118.788797:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4362 00000100:00080000:3.0:1767319118.788799:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319118.788803:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319118.788816:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982735104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319118.788838:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319118.788842:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319118.788847:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319118.788850:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319118.788854:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4362 00000100:00080000:3.0:1767319118.788857:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319118.788861:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319118.788876:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902680 x1853163982735488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319118.788880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982734976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319118.788910:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982734848:0@lo:400:kworker.0 00000100:00100000:0.0:1767319118.788912:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319118.788935:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982735104:0@lo:38:kworker.0 00000100:00100000:3.0:1767319118.788954:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319118.788970:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319118.788986:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319118.788997:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982735232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319118.788998:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982735488:0@lo:38:kworker.0 00000100:00100000:3.0:1767319118.789037:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319118.789049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319118.789086:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734720 00000100:00100000:0.0:1767319118.789093:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734720 00000100:00100000:0.0:1767319118.789122:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734976 00000100:00100000:0.0:1767319118.789125:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734976 00000100:00100000:0.0:1767319118.789133:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734848 00000100:00100000:0.0:1767319118.789135:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734848 00000100:00100000:0.0:1767319118.789162:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735488 00000100:00100000:0.0:1767319118.789165:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735488 00000100:00100000:0.0:1767319118.789179:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767319118.789190:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982734848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319118.789214:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211 00000100:00100000:3.0:1767319118.789221:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982734720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319118.789241:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319118.789243:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982734848:12345-0@lo:400:kworker.0 Request processed in 55us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319118.789252:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767319118.789308:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212 00000100:00100000:0.0:1767319118.789312:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982734976:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319118.789319:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982734720:12345-0@lo:38:kworker.0 Request processed in 94us (479us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767319118.789325:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319118.789329:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211 00000100:00100000:3.0:1767319118.789350:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1213 00000100:00100000:3.0:1767319118.789356:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982735488:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319118.789360:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982734976:12345-0@lo:38:kworker.0 Request processed in 43us (446us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767319118.789370:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319118.789371:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212 00000100:00100000:0.0:1767319118.789398:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735104 00000100:00100000:0.0:1767319118.789401:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735104 00000100:00100000:0.0:1767319118.789421:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 328 00000100:00100000:0.0:1767319118.789426:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982735104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319118.789434:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319118.789493:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735232 00000100:00100000:2.0:1767319118.789498:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735232 00000100:00100000:2.0:1767319118.789531:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 593 00000100:00100000:2.0:1767319118.789538:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982735232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319118.789626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982734592:0@lo:400:kworker.0 00000100:00100000:0.0:1767319118.790872:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982735104:12345-0@lo:38:kworker.0 Request processed in 1439us (1897us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319118.790884:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 328 00010000:00080000:0.0:1767319118.790971:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dea00 x1853163982734848/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319134 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319118.790985:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982734848:0@lo:400:kworker.0 00000100:00080000:0.0:1767319118.791189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319118.791196:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319118.791199:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982735488:0@lo:38:kworker.0 00000100:00080000:0.0:1767319118.791216:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319118.791220:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319118.791223:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982734976:0@lo:38:kworker.0 00000100:00080000:0.0:1767319118.791234:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319118.791238:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319118.791241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982735104:0@lo:38:kworker.0 00000100:00080000:0.0:1767319118.791251:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319118.791254:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319118.791257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982734720:0@lo:38:kworker.0 00000100:00100000:2.0:1767319118.791299:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982735232:12345-0@lo:400:kworker.0 Request processed in 1759us (2260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319118.791309:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 593 00000100:00080000:2.0:1767319118.791334:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319118.791340:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319118.791365:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982735232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319118.791406:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982735360:0@lo:400:kworker.0 00000100:00100000:0.0:1767319118.791441:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319118.791549:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735360 00000100:00100000:2.0:1767319118.791557:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735360 00000100:00100000:2.0:1767319118.791618:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 594 00000100:00100000:2.0:1767319118.791627:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982735360:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319118.791684:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982735360:12345-0@lo:400:kworker.0 Request processed in 59us (242us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319118.791689:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 594 00000100:00100000:2.0:1767319118.791717:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982735360:0@lo:400:kworker.0 00000100:00080000:0.0:1767319118.791878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319118.792244:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982735488:12345-0@lo:38:kworker.0 Request processed in 2884us (3194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319118.792252:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1213 00000100:00100000:1.0:1767319118.792374:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319118.792547:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982734592 00000100:00100000:3.0:1767319118.792553:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982734592 00000100:00100000:3.0:1767319118.792656:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924 00000100:00100000:3.0:1767319118.792664:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982734592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319118.792718:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982734592:12345-0@lo:400:kworker.0 Request processed in 56us (344us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319118.792725:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924 00010000:00080000:2.0:1767319118.792859:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddf80 x1853163982734592/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319134 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319118.792871:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982734592:0@lo:400:kworker.0 00000100:00080000:0.0:1767319118.793335:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319118.980513:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319120.005299:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319120.784523:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319120.784618:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319120.784644:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607168 00000100:00100000:1.0:1767319120.784656:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607168 00000100:00100000:3.0:1767319120.784693:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607040 00000100:00100000:3.0:1767319120.784698:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607040 00000100:00100000:3.0:1767319120.784746:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 925 00000100:00100000:3.0:1767319120.784758:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915607168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767319120.784769:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319120.784848:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915607168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (333us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319120.784856:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 925 00000100:00100000:3.0:1767319120.784873:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1214 00000100:00100000:3.0:1767319120.784878:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915607040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319120.784897:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319120.784921:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915607040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (304us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319120.784927:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1214 00010000:00010000:2.0:1767319121.028965:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319121.350205:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319121.350230:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982735744:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319121.350315:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319121.350707:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319121.350717:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982735616:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319121.350750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319121.356914:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735744 00000100:00100000:2.0:1767319121.356930:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735744 00000100:00080000:1.0:1767319121.357127:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319121.357471:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735616 00000100:00100000:2.0:1767319121.357484:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735616 00000100:00080000:1.0:1767319121.357548:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319121.363826:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 634 00000100:00100000:0.0:1767319121.363854:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982735744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319121.363893:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319121.363981:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982735744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (13665us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319121.363989:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 634 00000100:00100000:0.0:1767319121.364013:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 635 00000100:00100000:0.0:1767319121.364021:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982735616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319121.364051:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319121.364208:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982735744:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319121.364268:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982735616:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319121.364431:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982735616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (13340us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319121.364441:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 635 00010000:00010000:1.0:1767319122.052446:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319123.076499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319123.908382:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319123.908393:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319123.908505:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982735872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319123.908581:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319123.908679:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982735872 00000100:00100000:2.0:1767319123.908686:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982735872 00000100:00100000:2.0:1767319123.908752:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926 00000100:00100000:2.0:1767319123.908762:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982735872:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319123.908844:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319123.908949:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319123.908954:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319123.908957:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319123.908963:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4367 00000100:00080000:3.0:1767319123.908966:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319123.908971:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319123.908993:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903b80 x1853163982736000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319123.909033:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319123.909036:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319123.909038:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319123.909043:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4367 00000100:00080000:3.0:1767319123.909045:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319123.909049:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319123.909060:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902300 x1853163982736256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319123.909078:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319123.909082:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319123.909084:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319123.909088:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4367 00000100:00080000:3.0:1767319123.909090:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319123.909094:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319123.909114:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982736384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319123.909137:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319123.909148:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319123.909150:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319123.909154:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4367 00000100:00080000:3.0:1767319123.909157:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319123.909160:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319123.909170:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900000 x1853163982736768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319123.909203:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982736128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319123.909232:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319123.909251:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319123.909257:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982736512:0@lo:400:kworker.0 00000100:00100000:3.0:1767319123.909305:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319123.909337:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736512 00000100:00100000:3.0:1767319123.909341:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736512 00000100:00100000:3.0:1767319123.909375:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 595 00000100:00100000:3.0:1767319123.909381:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982736512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319123.909416:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982736512:12345-0@lo:400:kworker.0 Request processed in 34us (110us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319123.909423:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 595 00000100:00100000:3.0:1767319123.909463:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982736512:0@lo:400:kworker.0 00000100:00100000:0.0:1767319123.909542:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736128 00000100:00100000:0.0:1767319123.909546:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736128 00000100:00100000:0.0:1767319123.909583:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927 00000100:00100000:0.0:1767319123.909589:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982736128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319123.909628:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982736128:12345-0@lo:400:kworker.0 Request processed in 39us (396us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319123.909634:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927 00000100:00100000:0.0:1767319123.909652:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982736640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319123.909676:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319123.909701:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736640 00000100:00100000:0.0:1767319123.909705:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736640 00000100:00100000:0.0:1767319123.909728:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 596 00000100:00100000:0.0:1767319123.909735:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982736640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319123.909767:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982736640:12345-0@lo:400:kworker.0 Request processed in 32us (90us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319123.909774:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 596 00000100:00080000:0.0:1767319123.909787:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319123.909790:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319123.909792:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319123.909801:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982736000:0@lo:38:kworker.0 00000100:00100000:0.0:1767319123.909824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319123.909837:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982736256:0@lo:38:kworker.0 00000100:00100000:0.0:1767319123.909851:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319123.909861:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982736384:0@lo:38:kworker.0 00000100:00100000:0.0:1767319123.909874:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319123.909895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982736768:0@lo:38:kworker.0 00000100:00100000:0.0:1767319123.909906:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319123.909928:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736000 00000100:00100000:0.0:1767319123.909930:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736000 00000100:00100000:0.0:1767319123.909939:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736256 00000100:00100000:0.0:1767319123.909940:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736256 00000100:00100000:0.0:1767319123.909944:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736768 00000100:00100000:0.0:1767319123.909945:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736768 00000100:00100000:0.0:1767319123.909951:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215 00000100:00100000:0.0:1767319123.909954:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319123.909966:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319123.909989:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736000:12345-0@lo:38:kworker.0 Request processed in 33us (164us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319123.909995:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215 00000100:00100000:0.0:1767319123.910005:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1216 00000100:00100000:0.0:1767319123.910009:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319123.910017:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319123.910040:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736256:12345-0@lo:38:kworker.0 Request processed in 29us (186us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319123.910046:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1216 00000100:00100000:0.0:1767319123.910056:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217 00000100:00100000:0.0:1767319123.910060:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319123.910068:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319123.910084:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982736768:12345-0@lo:38:kworker.0 Request processed in 23us (177us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319123.910087:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217 00000100:00100000:0.0:1767319123.910099:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736384 00000100:00100000:0.0:1767319123.910101:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736384 00000100:00100000:0.0:1767319123.910113:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 329 00000100:00100000:0.0:1767319123.910115:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982736384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319123.910121:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319123.910135:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982736384:12345-0@lo:38:kworker.0 Request processed in 18us (260us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319123.910138:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 329 00000100:00100000:0.0:1767319123.910159:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982736640:0@lo:400:kworker.0 00000100:00080000:0.0:1767319123.910175:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319123.910178:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319123.910180:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982736000:0@lo:38:kworker.0 00000100:00080000:0.0:1767319123.910185:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319123.910187:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319123.910189:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982736256:0@lo:38:kworker.0 00000100:00080000:0.0:1767319123.910194:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319123.910196:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319123.910197:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982736384:0@lo:38:kworker.0 00000100:00080000:0.0:1767319123.910202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319123.910204:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319123.910205:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982736768:0@lo:38:kworker.0 00010000:00080000:0.0:1767319123.910223:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900a80 x1853163982736128/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319139 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319123.910229:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982736128:0@lo:400:kworker.0 00000100:00100000:2.0:1767319123.910700:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982735872:12345-0@lo:400:kworker.0 Request processed in 1937us (2115us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319123.910707:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926 00010000:00080000:1.0:1767319123.910778:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900700 x1853163982735872/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319139 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319123.910785:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982735872:0@lo:400:kworker.0 00010000:00010000:2.0:1767319124.101713:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319125.124470:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319125.901830:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319125.902414:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319125.907385:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607296 00000100:00100000:0.0:1767319125.907401:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607296 00000100:00100000:3.0:1767319125.907652:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607424 00000100:00100000:3.0:1767319125.907657:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607424 00000100:00100000:3.0:1767319125.907731:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 928 00000100:00100000:3.0:1767319125.907742:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915607424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319125.907837:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915607424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (5424us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319125.907845:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 928 00000100:00100000:3.0:1767319125.907864:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1218 00000100:00100000:3.0:1767319125.907870:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915607296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319125.907890:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319125.907916:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915607296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (6103us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319125.907922:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1218 00000100:00080000:1.0:1767319125.908480:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319126.149017:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319126.468826:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319126.468849:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982736896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319126.468920:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319126.469044:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982736896 00000100:00100000:2.0:1767319126.469052:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982736896 00000100:00100000:2.0:1767319126.469122:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 636 00000100:00100000:2.0:1767319126.469131:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982736896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319126.469158:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319126.469203:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982736896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319126.469209:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 636 00000100:00080000:3.0:1767319126.469428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319126.469465:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982736896:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319126.472380:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319126.478247:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982737024:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319126.478329:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319126.478521:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737024 00000100:00100000:2.0:1767319126.478530:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737024 00000100:00100000:2.0:1767319126.478590:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 637 00000100:00100000:2.0:1767319126.478601:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982737024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319126.478634:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319126.478690:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982737024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 91us (360us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319126.478697:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 637 00000100:00080000:3.0:1767319126.478941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319126.479054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982737024:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319127.175937:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319128.196709:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319129.028805:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319129.028817:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319129.028870:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319129.028875:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319129.028879:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319129.028888:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4372 00000100:00080000:0.0:1767319129.028892:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319129.028899:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319129.028920:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017800 x1853163982737280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319129.028953:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319129.028956:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319129.028958:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319129.028962:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4372 00000100:00080000:0.0:1767319129.028964:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319129.028968:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319129.028980:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddf80 x1853163982737536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319129.028988:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319129.028991:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319129.028993:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319129.028996:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4372 00000100:00080000:0.0:1767319129.028999:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319129.029003:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319129.029014:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dfb80 x1853163982737664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319129.029033:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982737280:0@lo:38:kworker.0 00000100:00080000:0.0:1767319129.029037:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319129.029043:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319129.029044:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319129.029048:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4372 00000100:00080000:0.0:1767319129.029050:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319129.029054:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319129.029065:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982738048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319129.029092:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982737792:0@lo:400:kworker.0 00000100:00100000:2.0:1767319129.029108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319129.029176:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982737920:0@lo:400:kworker.0 00000100:00100000:1.0:1767319129.029227:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982737152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.029355:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319129.029403:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319129.029409:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982737408:0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.029432:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319129.029487:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737792 00000100:00100000:0.0:1767319129.029495:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737792 00000100:00100000:0.0:1767319129.029563:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 597 00000100:00100000:0.0:1767319129.029573:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982737792:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.029714:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982737792:12345-0@lo:400:kworker.0 Request processed in 142us (358us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319129.029722:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 597 00000100:00100000:0.0:1767319129.029921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982737792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.029965:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737280 00000100:00100000:0.0:1767319129.029969:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737280 00000100:00100000:0.0:1767319129.029991:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737408 00000100:00100000:0.0:1767319129.029994:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737408 00000100:00100000:0.0:1767319129.030019:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1767319129.030026:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982737408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.030066:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982737408:12345-0@lo:400:kworker.0 Request processed in 40us (632us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319129.030072:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1767319129.030085:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1219 00000100:00100000:0.0:1767319129.030089:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982737280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319129.030107:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319129.030135:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982737280:12345-0@lo:38:kworker.0 Request processed in 44us (1029us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319129.030144:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1219 00010000:00080000:0.0:1767319129.030173:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017100 x1853163982737408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319145 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319129.030183:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982737408:0@lo:400:kworker.0 00000100:00080000:0.0:1767319129.030211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319129.030215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319129.031708:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319129.031717:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319129.031722:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982737280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319129.031733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982737536:0@lo:38:kworker.0 00000100:00100000:2.0:1767319129.031782:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319129.031823:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982737664:0@lo:38:kworker.0 00000100:00100000:2.0:1767319129.031849:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319129.031959:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982738048:0@lo:38:kworker.0 00000100:00100000:2.0:1767319129.031982:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319129.032020:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737664 00000100:00100000:2.0:1767319129.032026:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737664 00000100:00100000:2.0:1767319129.032056:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 330 00000100:00100000:2.0:1767319129.032062:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982737664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319129.032079:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319129.032116:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982737664:12345-0@lo:38:kworker.0 Request processed in 53us (266us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319129.032122:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 330 00000100:00100000:2.0:1767319129.032146:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737536 00000100:00100000:2.0:1767319129.032150:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737536 00000100:00100000:2.0:1767319129.032165:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738048 00000100:00100000:2.0:1767319129.032168:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738048 00000100:00100000:2.0:1767319129.032183:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220 00000100:00100000:2.0:1767319129.032188:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982737536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319129.032197:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319129.032221:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982737536:12345-0@lo:38:kworker.0 Request processed in 32us (438us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319129.032227:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220 00000100:00100000:2.0:1767319129.032238:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221 00000100:00100000:2.0:1767319129.032242:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982738048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319129.032250:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319129.032292:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982738048:12345-0@lo:38:kworker.0 Request processed in 49us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319129.032298:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221 00000100:00080000:2.0:1767319129.032327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319129.032330:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319129.032334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982738048:0@lo:38:kworker.0 00000100:00080000:2.0:1767319129.032342:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319129.032344:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319129.032346:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982737536:0@lo:38:kworker.0 00000100:00080000:2.0:1767319129.032353:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319129.032356:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319129.032359:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982737664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319129.034753:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319129.034822:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737920 00000100:00100000:0.0:1767319129.034831:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737920 00000100:00100000:0.0:1767319129.034911:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 598 00000100:00100000:0.0:1767319129.034921:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982737920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.034984:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982737920:12345-0@lo:400:kworker.0 Request processed in 65us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319129.034992:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 598 00000100:00100000:1.0:1767319129.036311:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319129.036488:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319129.037064:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982737152 00000100:00100000:0.0:1767319129.037074:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982737152 00000100:00100000:0.0:1767319129.037141:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 930 00000100:00100000:0.0:1767319129.037150:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982737152:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319129.037243:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319129.037833:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982737920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319129.038634:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982737152:12345-0@lo:400:kworker.0 Request processed in 1483us (2323us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319129.038646:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 930 00010000:00080000:1.0:1767319129.038743:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014e00 x1853163982737152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319145 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319129.038756:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982737152:0@lo:400:kworker.0 00010000:00010000:1.0:1767319129.220543:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319130.244536:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319131.019767:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319131.020348:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319131.020725:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607680 00000100:00100000:0.0:1767319131.020737:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607680 00000100:00100000:3.0:1767319131.020755:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607552 00000100:00100000:3.0:1767319131.020761:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607552 00000100:00080000:2.0:1767319131.020827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319131.020986:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 931 00000100:00100000:3.0:1767319131.020999:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915607680:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319131.021091:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915607680:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (1339us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319131.021098:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1222 00000100:00100000:3.0:1767319131.021107:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 931 00000100:00100000:0.0:1767319131.021109:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915607552:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319131.021324:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319131.021356:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915607552:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 248us (1008us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319131.021363:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1222 00010000:00010000:1.0:1767319131.273674:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319131.588829:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319131.588861:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982738176:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319131.588931:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319131.589010:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319131.589022:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982738304:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319131.589064:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319131.597863:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738176 00000100:00100000:2.0:1767319131.597880:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738176 00000100:00100000:2.0:1767319131.597973:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738304 00000100:00100000:2.0:1767319131.597977:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738304 00000100:00100000:2.0:1767319131.597998:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 638 00000100:00100000:2.0:1767319131.598006:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982738176:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319131.598039:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319131.598108:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 639 00000100:00100000:1.0:1767319131.598131:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982738304:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319131.598139:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982738176:12345-0@lo:13:osp-pre-0-1.0 Request processed in 129us (9204us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319131.598147:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 638 00002000:00000024:1.0:1767319131.598152:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319131.598202:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982738304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (9137us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319131.598210:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 639 00000100:00100000:2.0:1767319131.598236:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982738176:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319131.598255:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982738304:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319131.598761:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319131.598766:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319132.292529:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319133.316524:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319134.149513:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319134.149524:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319134.149600:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319134.149604:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319134.149608:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319134.149617:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4377 00000100:00080000:3.0:1767319134.149622:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319134.149631:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319134.149657:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163982738560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319134.149698:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319134.149702:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319134.149703:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319134.149707:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4377 00000100:00080000:3.0:1767319134.149709:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319134.149713:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319134.149724:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163982738816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319134.149732:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319134.149735:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319134.149736:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319134.149739:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4377 00000100:00080000:3.0:1767319134.149742:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319134.149745:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319134.149758:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902300 x1853163982738944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319134.149778:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319134.149782:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319134.149783:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319134.149786:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4377 00000100:00080000:3.0:1767319134.149788:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319134.149792:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319134.149801:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902680 x1853163982739328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319134.149881:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982739200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319134.149967:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319134.150053:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319134.150059:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982738432:0@lo:400:kworker.0 00000100:00100000:0.0:1767319134.150085:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319134.150170:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982738560:0@lo:38:kworker.0 00000100:00100000:3.0:1767319134.150190:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739200 00000100:00100000:3.0:1767319134.150200:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739200 00000100:00100000:2.0:1767319134.150205:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319134.150229:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982738816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319134.150251:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319134.150295:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 599 00000100:00100000:2.0:1767319134.150299:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982738944:0@lo:38:kworker.0 00000100:00100000:3.0:1767319134.150309:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982739200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319134.150330:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319134.150372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982739328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319134.150387:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982739200:12345-0@lo:400:kworker.0 Request processed in 81us (418us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319134.150401:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 599 00000100:00100000:2.0:1767319134.150403:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319134.150431:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319134.150443:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738432 00000100:00100000:2.0:1767319134.150448:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738432 00000100:00100000:3.0:1767319134.150460:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738560 00000100:00100000:3.0:1767319134.150466:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738560 00000100:00100000:2.0:1767319134.150488:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738816 00000100:00100000:3.0:1767319134.150489:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739328 00000100:00100000:2.0:1767319134.150495:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738816 00000100:00100000:3.0:1767319134.150495:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739328 00000100:00100000:3.0:1767319134.150515:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 932 00000100:00100000:2.0:1767319134.150523:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223 00000100:00100000:3.0:1767319134.150528:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982738432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319134.150532:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982738560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319134.150553:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319134.150568:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982738432:12345-0@lo:400:kworker.0 Request processed in 44us (481us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319134.150575:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 932 00000100:00100000:3.0:1767319134.150593:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224 00000100:00100000:2.0:1767319134.150598:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982738560:12345-0@lo:38:kworker.0 Request processed in 64us (390us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319134.150601:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982739328:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319134.150612:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223 00010000:00020000:3.0:1767319134.150618:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319134.150629:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1225 00000100:00100000:2.0:1767319134.150635:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982738816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319134.150645:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319134.150648:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982739328:12345-0@lo:38:kworker.0 Request processed in 45us (248us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319134.150656:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224 00000100:00100000:2.0:1767319134.150678:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982738816:12345-0@lo:38:kworker.0 Request processed in 41us (425us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319134.150681:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982738688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319134.150689:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1225 00000100:00100000:2.0:1767319134.150712:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738944 00000100:00100000:2.0:1767319134.150715:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738944 00000100:00100000:3.0:1767319134.150720:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319134.150740:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 331 00000100:00100000:3.0:1767319134.150744:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319134.150747:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982738944:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319134.150754:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982739072:0@lo:400:kworker.0 00010000:00020000:2.0:1767319134.150760:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319134.150787:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319134.150799:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982738944:12345-0@lo:38:kworker.0 Request processed in 50us (468us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319134.150806:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 331 00000100:00100000:3.0:1767319134.150817:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739072 00000100:00100000:3.0:1767319134.150822:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739072 00000100:00100000:2.0:1767319134.150829:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982738688 00000100:00100000:2.0:1767319134.150834:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982738688 00000100:00100000:3.0:1767319134.150855:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 600 00000100:00100000:3.0:1767319134.150865:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982739072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319134.150870:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933 00000100:00100000:2.0:1767319134.150879:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982738688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319134.150907:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982739072:12345-0@lo:400:kworker.0 Request processed in 43us (119us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319134.150914:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 600 00000100:00100000:2.0:1767319134.150920:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982738688:12345-0@lo:400:kworker.0 Request processed in 41us (200us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319134.150928:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933 00000100:00100000:3.0:1767319134.150951:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982739072:0@lo:400:kworker.0 00000100:00080000:2.0:1767319134.150962:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319134.150967:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319134.150968:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900000 x1853163982738688/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319150 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319134.150978:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982738688:0@lo:400:kworker.0 00000100:00080000:2.0:1767319134.150990:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319134.150997:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319134.151001:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982738560:0@lo:38:kworker.0 00000100:00080000:2.0:1767319134.151011:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319134.151015:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319134.151018:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982738816:0@lo:38:kworker.0 00000100:00080000:2.0:1767319134.151025:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319134.151028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319134.151030:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982738944:0@lo:38:kworker.0 00000100:00080000:2.0:1767319134.151037:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319134.151040:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319134.151042:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982739328:0@lo:38:kworker.0 00010000:00080000:0.0:1767319134.154331:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900700 x1853163982738432/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319150 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319134.154365:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982738432:0@lo:400:kworker.0 00000100:00100000:0.0:1767319134.154384:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982739200:0@lo:400:kworker.0 00010000:00010000:1.0:1767319134.343076:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767319135.120502:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915607808 offset 0 length 520: 4 00000100:00100000:2.0:1767319135.121305:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319135.121363:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319135.121585:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915607936 00000100:00100000:3.0:1767319135.121595:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915607936 00000100:00100000:3.0:1767319135.121636:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608064 00000100:00100000:3.0:1767319135.121639:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608064 00000100:00100000:2.0:1767319135.121667:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1226 00000100:00100000:2.0:1767319135.121676:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915607936:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:3.0:1767319135.121693:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 934 00000100:00100000:3.0:1767319135.121700:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915608064:12345-192.168.204.7@tcp:400:kworker.0 00010000:00020000:2.0:1767319135.121701:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319135.121780:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319135.122106:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915608064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 405us (741us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319135.122123:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 934 00000100:00100000:2.0:1767319135.123729:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915607936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 86us (460us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319135.123752:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1226 00010000:00010000:1.0:1767319135.364604:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319136.388456:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319136.708714:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319136.708731:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982739456:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319136.708802:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319136.708887:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319136.708890:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739456 00000100:00100000:1.0:1767319136.708899:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982739584:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319136.708900:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739456 00000100:00100000:1.0:1767319136.708932:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319136.708965:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739584 00000100:00100000:1.0:1767319136.708970:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739584 00000100:00100000:3.0:1767319136.708994:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 640 00000100:00100000:1.0:1767319136.709001:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 641 00000100:00100000:3.0:1767319136.709007:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982739456:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319136.709014:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982739584:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319136.709020:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319136.709026:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767319136.709040:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:3.0:1767319136.709041:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319136.709100:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982739584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319136.709113:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 641 00000100:00100000:3.0:1767319136.709181:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982739456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 178us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319136.709191:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 640 00000100:00100000:2.0:1767319136.709219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982739584:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319136.713552:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982739456:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319137.418944:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319138.249015:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319113/real 1767319113] req@ffff9346076fa300 x1853163982734208/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319138 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319138.249051:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076fa300 x1853163982734208/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319138 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319138.249072:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319138.249101:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319138.249106:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982734208:0@lo:250:kworker.0 00000100:00080000:0.0:1767319138.249358:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319138.249364:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319138.249367:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319138.249372:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4362 00000100:00080000:0.0:1767319138.249376:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319138.249389:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319138.249423:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982739712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319138.249487:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319138.249490:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319138.249492:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319138.249496:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4382 00000100:00080000:0.0:1767319138.249498:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319138.249502:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319138.249517:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014e00 x1853163982739968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319138.249546:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319138.249549:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319138.249551:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319138.249554:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4382 00000100:00080000:0.0:1767319138.249557:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319138.249561:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319138.249572:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163982740224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319138.249581:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319138.249584:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319138.249586:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319138.249589:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4382 00000100:00080000:0.0:1767319138.249591:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319138.249595:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319138.249606:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cca80 x1853163982740352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319138.249641:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319138.249645:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319138.249647:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319138.249650:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4382 00000100:00080000:0.0:1767319138.249652:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319138.249655:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319138.249665:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163982740736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319138.249688:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982740096:0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.249755:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319138.249812:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319138.249818:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982740480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.249849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319138.249889:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740480 00000100:00100000:0.0:1767319138.249897:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740480 00000100:00100000:0.0:1767319138.249947:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 601 00000100:00100000:0.0:1767319138.249956:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982740480:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.250224:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982740480:12345-0@lo:400:kworker.0 Request processed in 47us (152us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319138.250234:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 601 00000100:00100000:0.0:1767319138.250313:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982740480:0@lo:400:kworker.0 00000100:00100000:1.0:1767319138.250400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982739840:0@lo:400:kworker.0 00000100:00100000:1.0:1767319138.250433:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319138.250470:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319138.250475:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982740608:0@lo:400:kworker.0 00000100:00100000:1.0:1767319138.250499:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319138.250523:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740608 00000100:00100000:1.0:1767319138.250527:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740608 00000100:00100000:1.0:1767319138.250553:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 602 00000100:00100000:1.0:1767319138.250558:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982740608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319138.250589:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982740608:12345-0@lo:400:kworker.0 Request processed in 31us (89us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319138.250595:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 602 00000100:00100000:1.0:1767319138.250632:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982740608:0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.250928:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740096 00000100:00100000:0.0:1767319138.250933:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740096 00000100:00100000:0.0:1767319138.250968:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739840 00000100:00100000:0.0:1767319138.250972:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739840 00000100:00100000:0.0:1767319138.251000:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935 00000100:00100000:0.0:1767319138.251093:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982740096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.251149:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982740096:12345-0@lo:400:kworker.0 Request processed in 60us (1394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319138.251156:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935 00000100:00100000:0.0:1767319138.251170:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936 00000100:00100000:0.0:1767319138.251177:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982739840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319138.251206:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982739840:12345-0@lo:400:kworker.0 Request processed in 30us (772us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319138.251211:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936 00010000:00080000:0.0:1767319138.251236:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017800 x1853163982740096/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319138.251246:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982740096:0@lo:400:kworker.0 00000100:00080000:2.0:1767319138.251426:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319138.251431:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319138.251434:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319138.251446:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982739712:0@lo:250:kworker.0 00000400:00000100:2.0:1767319138.251472:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982739712 offset 0 length 520: 4 00000100:00100000:2.0:1767319138.251494:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982739968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319138.251517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319138.251546:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982740224:0@lo:38:kworker.0 00000100:00100000:2.0:1767319138.251574:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319138.251603:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982740352:0@lo:38:kworker.0 00000100:00100000:2.0:1767319138.251647:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319138.251666:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982740736:0@lo:38:kworker.0 00000100:00100000:2.0:1767319138.251689:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319138.252161:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982739968 00000100:00100000:3.0:1767319138.252167:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982739968 00000100:00100000:3.0:1767319138.252198:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740224 00000100:00100000:3.0:1767319138.252201:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740224 00000100:00100000:3.0:1767319138.252209:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740736 00000100:00100000:3.0:1767319138.252211:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740736 00000100:00100000:3.0:1767319138.252220:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227 00000100:00100000:3.0:1767319138.252225:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982739968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319138.252244:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319138.252330:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982739968:12345-0@lo:38:kworker.0 Request processed in 103us (812us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319138.252337:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227 00000100:00100000:3.0:1767319138.252352:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1228 00000100:00100000:3.0:1767319138.252356:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982740224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319138.252365:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319138.252389:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982740224:12345-0@lo:38:kworker.0 Request processed in 31us (814us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319138.252394:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1228 00000100:00100000:3.0:1767319138.252406:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229 00000100:00100000:3.0:1767319138.252410:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982740736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319138.252418:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319138.252451:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982740736:12345-0@lo:38:kworker.0 Request processed in 39us (761us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319138.252457:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229 00000100:00080000:3.0:1767319138.252514:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319138.252519:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319138.252523:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982739968:0@lo:38:kworker.0 00000100:00080000:3.0:1767319138.252530:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319138.252533:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319138.252536:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982740224:0@lo:38:kworker.0 00000100:00080000:3.0:1767319138.252543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319138.252546:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319138.252548:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982740736:0@lo:38:kworker.0 00000100:00100000:2.0:1767319138.252738:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740352 00000100:00100000:2.0:1767319138.252744:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740352 00000100:00100000:2.0:1767319138.252776:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 332 00000100:00100000:2.0:1767319138.252781:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982740352:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319138.252791:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319138.252827:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982740352:12345-0@lo:38:kworker.0 Request processed in 46us (1179us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319138.252834:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 332 00000100:00080000:2.0:1767319138.252858:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319138.252862:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319138.252865:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982740352:0@lo:38:kworker.0 00010000:00080000:1.0:1767319138.253242:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382de300 x1853163982739840/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319138.253255:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982739840:0@lo:400:kworker.0 00010000:00010000:1.0:1767319138.436872:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319139.460471:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319140.237664:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319140.237950:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319140.244255:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608320 00000100:00100000:0.0:1767319140.244303:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608320 00000100:00100000:0.0:1767319140.244385:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608192 00000100:00100000:0.0:1767319140.244390:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608192 00000100:00100000:0.0:1767319140.244426:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 937 00000100:00100000:0.0:1767319140.244440:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915608320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319140.244449:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1230 00000100:00100000:3.0:1767319140.244460:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319140.244488:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319140.244532:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915608320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 96us (6878us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319140.244542:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 937 00000100:00080000:0.0:1767319140.244588:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319140.254840:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 10368us (16877us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319140.254854:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1230 00010000:00010000:1.0:1767319140.485098:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319141.508556:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319141.831087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982740864:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319141.831164:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319141.831252:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740864 00000100:00100000:3.0:1767319141.831259:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740864 00000100:00100000:3.0:1767319141.831334:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 642 00000100:00100000:3.0:1767319141.831344:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982740864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319141.831375:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319141.831418:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982740864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319141.831425:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 642 00000100:00080000:3.0:1767319141.831447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319141.831487:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982740864:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319141.831571:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982740992:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319141.831601:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319141.831919:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982740992 00000100:00100000:3.0:1767319141.831923:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982740992 00000100:00100000:3.0:1767319141.831964:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 643 00000100:00100000:3.0:1767319141.831971:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982740992:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319141.831988:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319141.832022:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982740992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (420us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319141.832029:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 643 00000100:00100000:3.0:1767319141.832062:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982740992:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319141.833290:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767319142.534913:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319143.364411:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319143.364424:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319143.364524:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319143.364529:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319143.364533:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319143.364541:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4386 00000100:00080000:0.0:1767319143.364546:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319143.364553:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319143.364568:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982741120:0@lo:400:kworker.0 00000100:00100000:0.0:1767319143.364575:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5c00 x1853163982741248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319143.364615:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319143.364619:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319143.364621:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319143.364624:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4386 00000100:00080000:0.0:1767319143.364626:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319143.364630:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319143.364640:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982741504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319143.364640:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319143.364648:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319143.364651:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319143.364653:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319143.364655:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4386 00000100:00080000:0.0:1767319143.364658:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319143.364661:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319143.364671:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982741632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319143.364685:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741120 00000100:00080000:0.0:1767319143.364690:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319143.364692:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741120 00000100:00080000:0.0:1767319143.364693:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319143.364695:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319143.364698:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4386 00000100:00080000:0.0:1767319143.364700:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319143.364703:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319143.364712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6d80 x1853163982742016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319143.364748:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982741376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.364756:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 938 00000100:00100000:3.0:1767319143.364765:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982741120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.364813:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982741120:12345-0@lo:400:kworker.0 Request processed in 49us (171us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319143.364820:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 938 00000100:00100000:3.0:1767319143.364850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982741760:0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.364878:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319143.364925:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767319143.364925:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7b80 x1853163982741120/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319143.364938:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982741120:0@lo:400:kworker.0 00000100:00100000:0.0:1767319143.364953:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741376 00000100:00100000:0.0:1767319143.364957:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741376 00000100:00100000:3.0:1767319143.364957:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319143.364962:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982741888:0@lo:400:kworker.0 00000100:00100000:0.0:1767319143.364982:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939 00000100:00100000:3.0:1767319143.364983:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319143.364988:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982741376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.365003:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741760 00000100:00100000:3.0:1767319143.365007:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741760 00000100:00100000:0.0:1767319143.365019:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982741376:12345-0@lo:400:kworker.0 Request processed in 31us (95us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319143.365025:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939 00000100:00100000:3.0:1767319143.365029:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741888 00000100:00100000:3.0:1767319143.365032:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741888 00010000:00080000:0.0:1767319143.365048:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982741376/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319143.365050:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 603 00000100:00100000:3.0:1767319143.365056:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982741760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319143.365058:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982741376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.365087:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982741760:12345-0@lo:400:kworker.0 Request processed in 31us (207us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319143.365093:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 603 00000100:00100000:3.0:1767319143.365105:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 604 00000100:00100000:3.0:1767319143.365110:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982741888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.365134:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982741888:12345-0@lo:400:kworker.0 Request processed in 24us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319143.365140:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 604 00000100:00100000:3.0:1767319143.365163:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982741760:0@lo:400:kworker.0 00000100:00100000:3.0:1767319143.365195:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982741888:0@lo:400:kworker.0 00000100:00080000:1.0:1767319143.365380:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319143.365384:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319143.365387:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319143.365391:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982741248:0@lo:38:kworker.0 00000100:00100000:2.0:1767319143.365420:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319143.365573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982741504:0@lo:38:kworker.0 00000100:00100000:2.0:1767319143.365601:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319143.365630:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982741632:0@lo:38:kworker.0 00000100:00100000:2.0:1767319143.365655:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319143.365678:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982742016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319143.365699:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319143.365723:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741632 00000100:00100000:1.0:1767319143.365728:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741632 00000100:00100000:2.0:1767319143.365729:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741248 00000100:00100000:2.0:1767319143.365731:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741248 00000100:00100000:2.0:1767319143.365749:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982741504 00000100:00100000:1.0:1767319143.365751:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 333 00000100:00100000:2.0:1767319143.365751:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982741504 00000100:00100000:1.0:1767319143.365756:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982741632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319143.365774:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319143.365813:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982741632:12345-0@lo:38:kworker.0 Request processed in 55us (156us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319143.365819:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 333 00000100:00080000:1.0:1767319143.365850:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319143.365855:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319143.365858:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982741632:0@lo:38:kworker.0 00000100:00100000:2.0:1767319143.366127:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742016 00000100:00100000:2.0:1767319143.366132:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742016 00000100:00100000:1.0:1767319143.366819:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1231 00000100:00100000:1.0:1767319143.366826:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982741248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319143.366840:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319143.366891:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982741248:12345-0@lo:38:kworker.0 Request processed in 64us (1469us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319143.366899:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1231 00000100:00100000:1.0:1767319143.366916:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232 00000100:00100000:1.0:1767319143.366920:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982741504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319143.366930:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319143.366953:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982741504:12345-0@lo:38:kworker.0 Request processed in 31us (1351us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319143.366958:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232 00000100:00100000:1.0:1767319143.366969:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233 00000100:00100000:1.0:1767319143.366973:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982742016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319143.366982:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319143.367005:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982742016:12345-0@lo:38:kworker.0 Request processed in 31us (1305us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319143.367011:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233 00000100:00080000:1.0:1767319143.367051:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319143.367055:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319143.367059:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982741248:0@lo:38:kworker.0 00000100:00080000:1.0:1767319143.367068:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319143.367071:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319143.367074:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982741504:0@lo:38:kworker.0 00000100:00080000:1.0:1767319143.367082:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319143.367085:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319143.367087:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982742016:0@lo:38:kworker.0 00010000:00010000:1.0:1767319143.568082:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319144.580520:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319145.356619:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319145.356714:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319145.357648:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608448 00000100:00100000:1.0:1767319145.357661:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608448 00000100:00100000:1.0:1767319145.357713:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608576 00000100:00100000:1.0:1767319145.357716:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608576 00000100:00100000:1.0:1767319145.357776:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 940 00000100:00100000:1.0:1767319145.357789:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915608576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319145.357883:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915608576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (1168us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319145.357892:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 940 00000100:00100000:1.0:1767319145.357921:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1234 00000100:00100000:1.0:1767319145.357927:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608448:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319145.357949:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319145.357978:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 50us (1373us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319145.357985:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1234 00000100:00080000:3.0:1767319145.358419:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319145.606388:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319146.631497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319146.948615:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982742144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319146.948684:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319146.949019:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742144 00000100:00100000:1.0:1767319146.949026:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742144 00000100:00100000:1.0:1767319146.949095:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 644 00000100:00100000:1.0:1767319146.949115:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982742144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319146.949147:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319146.949196:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982742144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319146.949203:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 644 00000100:00080000:3.0:1767319146.949331:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319146.949368:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982742144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319146.950126:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319146.950138:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982742272:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319146.950179:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319146.950379:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742272 00000100:00100000:1.0:1767319146.950385:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742272 00000100:00100000:1.0:1767319146.950438:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 645 00000100:00100000:1.0:1767319146.950445:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982742272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319146.950468:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319146.950523:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982742272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319146.950530:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 645 00000100:00100000:2.0:1767319146.950685:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982742272:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319146.950738:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319147.655199:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319148.484873:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319148.484885:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319148.484952:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319148.484958:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319148.484963:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319148.484971:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4391 00000100:00080000:0.0:1767319148.484975:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319148.484983:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319148.485006:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5180 x1853163982742528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319148.485050:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319148.485054:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319148.485056:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319148.485060:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4391 00000100:00080000:0.0:1767319148.485062:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319148.485066:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319148.485085:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4380 x1853163982742784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319148.485094:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319148.485097:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319148.485099:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319148.485102:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4391 00000100:00080000:0.0:1767319148.485104:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319148.485107:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319148.485116:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982742912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319148.485142:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319148.485145:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319148.485147:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319148.485150:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4391 00000100:00080000:0.0:1767319148.485152:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319148.485155:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319148.485162:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5c00 x1853163982743296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319148.485186:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982742656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319148.485251:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319148.485303:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319148.485308:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982743040:0@lo:400:kworker.0 00000100:00100000:0.0:1767319148.485340:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319148.485420:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743040 00000100:00100000:1.0:1767319148.485427:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743040 00000100:00100000:1.0:1767319148.485486:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 605 00000100:00100000:1.0:1767319148.485495:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982743040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319148.485547:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982743040:12345-0@lo:400:kworker.0 Request processed in 53us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319148.485554:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 605 00000100:00100000:1.0:1767319148.485583:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982743168:0@lo:400:kworker.0 00000100:00100000:1.0:1767319148.485607:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319148.485637:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319148.485643:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982742400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319148.485664:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319148.485686:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743168 00000100:00100000:1.0:1767319148.485690:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743168 00000100:00100000:1.0:1767319148.485713:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 606 00000100:00100000:2.0:1767319148.485714:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742656 00000100:00100000:2.0:1767319148.485718:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742656 00000100:00100000:1.0:1767319148.485719:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982743168:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319148.485746:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742400 00000100:00100000:2.0:1767319148.485750:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742400 00000100:00100000:1.0:1767319148.485752:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982743168:12345-0@lo:400:kworker.0 Request processed in 33us (145us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319148.485758:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 606 00000100:00100000:2.0:1767319148.485773:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941 00000100:00100000:2.0:1767319148.485780:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982742656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319148.485806:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982743168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319148.485811:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982742656:12345-0@lo:400:kworker.0 Request processed in 33us (560us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319148.485816:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941 00000100:00100000:2.0:1767319148.485827:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 942 00000100:00100000:2.0:1767319148.485833:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982742400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319148.485834:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982743040:0@lo:400:kworker.0 00010000:00080000:1.0:1767319148.485844:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7800 x1853163982742656/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319164 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319148.485854:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982742656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319148.485858:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982742400:12345-0@lo:400:kworker.0 Request processed in 27us (195us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319148.485864:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 942 00000100:00080000:0.0:1767319148.485920:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319148.485924:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319148.485926:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319148.485997:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982742528:0@lo:38:kworker.0 00000100:00100000:3.0:1767319148.486027:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767319148.492042:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982742400/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319164 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319148.492062:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982742400:0@lo:400:kworker.0 00000100:00100000:3.0:1767319148.505158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982742784:0@lo:38:kworker.0 00000100:00100000:3.0:1767319148.505233:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319148.507597:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982742912:0@lo:38:kworker.0 00000100:00100000:3.0:1767319148.507672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319148.507704:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982743296:0@lo:38:kworker.0 00000100:00100000:3.0:1767319148.507730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319148.507861:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742528 00000100:00100000:0.0:1767319148.507871:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742528 00000100:00100000:0.0:1767319148.507913:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742784 00000100:00100000:0.0:1767319148.507916:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742784 00000100:00100000:0.0:1767319148.507936:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767319148.507944:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982742528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319148.507968:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319148.508013:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982742528:12345-0@lo:38:kworker.0 Request processed in 69us (21985us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319148.508020:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767319148.508037:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236 00000100:00100000:0.0:1767319148.508041:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982742784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319148.508050:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319148.508084:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982742784:12345-0@lo:38:kworker.0 Request processed in 31us (2841us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319148.508090:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236 00000100:00100000:1.0:1767319148.515666:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982742912 00000100:00100000:1.0:1767319148.515688:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982742912 00000100:00100000:1.0:1767319148.515753:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 334 00000100:00100000:1.0:1767319148.515762:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982742912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319148.515941:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319148.516025:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982742912:12345-0@lo:38:kworker.0 Request processed in 263us (8352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319148.516032:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 334 00000100:00080000:3.0:1767319148.516188:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319148.516195:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319148.516199:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982742528:0@lo:38:kworker.0 00000100:00080000:3.0:1767319148.516211:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319148.516215:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319148.516218:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982742784:0@lo:38:kworker.0 00000100:00080000:3.0:1767319148.516238:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319148.516243:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319148.516245:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982742912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319148.519363:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743296 00000100:00100000:1.0:1767319148.519383:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743296 00000100:00100000:1.0:1767319148.519452:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1237 00000100:00100000:1.0:1767319148.519461:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982743296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319148.519484:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319148.519587:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982743296:12345-0@lo:38:kworker.0 Request processed in 127us (11856us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319148.519594:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1237 00000100:00080000:3.0:1767319148.520340:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319148.520347:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319148.520351:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982743296:0@lo:38:kworker.0 00010000:00010000:0.0:1767319148.676511:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319149.702681:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319150.479827:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319150.479943:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319150.480097:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608832 00000100:00100000:1.0:1767319150.480108:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608832 00000100:00100000:1.0:1767319150.480185:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608704 00000100:00100000:1.0:1767319150.480189:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608704 00000100:00100000:1.0:1767319150.480208:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 943 00000100:00100000:2.0:1767319150.480222:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1238 00000100:00100000:1.0:1767319150.480223:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915608832:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319150.480235:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608704:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319150.480259:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319150.480322:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319150.480571:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915608704:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 335us (625us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319150.480583:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1238 00000100:00100000:1.0:1767319150.482861:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915608832:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 2631us (3036us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319150.482877:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 943 00010000:00010000:1.0:1767319150.725539:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319151.748922:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319152.068532:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319152.068559:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982743424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319152.068632:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319152.068909:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743424 00000100:00100000:0.0:1767319152.068918:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743424 00000100:00080000:3.0:1767319152.069072:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319152.069624:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982743552:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319152.069666:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319152.077079:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743552 00000100:00100000:0.0:1767319152.077096:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743552 00000100:00100000:0.0:1767319152.077172:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 646 00000100:00100000:0.0:1767319152.077185:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982743424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319152.077229:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319152.077314:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 647 00000100:00100000:1.0:1767319152.077338:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982743552:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319152.077343:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982743424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 162us (8712us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319152.077350:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 646 00002000:00000024:1.0:1767319152.077357:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767319152.077379:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319152.077397:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982743552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (7730us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319152.077403:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 647 00000100:00100000:3.0:1767319152.077833:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982743424:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319152.077871:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982743552:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319152.772685:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319153.604746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319153.604757:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319153.604823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319153.604827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319153.604831:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319153.604840:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4397 00000100:00080000:2.0:1767319153.604845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319153.604852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319153.604875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962a00 x1853163982743808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319153.604882:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319153.604905:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982743680:0@lo:400:kworker.0 00000100:00080000:2.0:1767319153.604923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319153.604928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319153.604930:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319153.604934:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4397 00000100:00080000:2.0:1767319153.604937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319153.604941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319153.604953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163982744064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319153.604962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319153.604965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319153.604966:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319153.604969:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4397 00000100:00080000:2.0:1767319153.604972:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319153.604975:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319153.605003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960e00 x1853163982744192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319153.605007:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319153.605034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319153.605037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319153.605070:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982744448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605096:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319153.605141:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982744320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605164:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319153.605181:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319153.605187:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982743936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605207:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319153.605248:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743680 00000100:00100000:0.0:1767319153.605256:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743680 00000100:00100000:0.0:1767319153.605343:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743936 00000100:00100000:0.0:1767319153.605347:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743936 00000100:00100000:0.0:1767319153.605378:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944 00000100:00100000:1.0:1767319153.605387:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945 00000100:00100000:0.0:1767319153.605388:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982743680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319153.605404:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982743936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605468:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982743680:12345-0@lo:400:kworker.0 Request processed in 69us (452us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319153.605476:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982743936:12345-0@lo:400:kworker.0 Request processed in 76us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319153.605481:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944 00000100:00100000:1.0:1767319153.605490:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945 00000100:00080000:0.0:1767319153.605510:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319153.605522:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744448 00000100:00100000:1.0:1767319153.605527:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744448 00010000:00080000:0.0:1767319153.605551:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960700 x1853163982743680/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319169 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319153.605565:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982743680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319153.605567:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744320 00000100:00100000:1.0:1767319153.605572:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744320 00000100:00100000:1.0:1767319153.605595:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 607 00000100:00100000:1.0:1767319153.605605:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982744448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605610:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 608 00000100:00100000:0.0:1767319153.605625:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982744320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319153.605661:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982744448:12345-0@lo:400:kworker.0 Request processed in 57us (562us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319153.605671:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 607 00000100:00100000:0.0:1767319153.605672:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982744320:12345-0@lo:400:kworker.0 Request processed in 49us (504us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319153.605681:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 608 00010000:00080000:1.0:1767319153.605716:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f961500 x1853163982743936/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319169 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319153.605728:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982743936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319153.605739:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982744448:0@lo:400:kworker.0 00000100:00100000:1.0:1767319153.605747:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982744320:0@lo:400:kworker.0 00000100:00080000:2.0:1767319153.605882:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319153.605887:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4397 00000100:00080000:2.0:1767319153.605890:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319153.605894:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319153.605912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982744576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319153.606666:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982743808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319153.606713:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319153.606757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982744064:0@lo:38:kworker.0 00000100:00100000:3.0:1767319153.606777:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319153.606808:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982744192:0@lo:38:kworker.0 00000100:00100000:3.0:1767319153.606831:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319153.606860:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982744576:0@lo:38:kworker.0 00000100:00100000:3.0:1767319153.606880:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319153.606902:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982743808 00000100:00100000:3.0:1767319153.606908:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982743808 00000100:00100000:3.0:1767319153.606932:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744064 00000100:00100000:3.0:1767319153.606935:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744064 00000100:00100000:3.0:1767319153.607029:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744576 00000100:00100000:3.0:1767319153.607033:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744576 00000100:00100000:3.0:1767319153.607046:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239 00000100:00100000:3.0:1767319153.607052:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982743808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319153.607070:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319153.607102:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982743808:12345-0@lo:38:kworker.0 Request processed in 50us (389us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319153.607109:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239 00000100:00100000:3.0:1767319153.607121:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1240 00000100:00100000:3.0:1767319153.607125:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982744064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319153.607133:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319153.607150:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982744064:12345-0@lo:38:kworker.0 Request processed in 24us (372us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319153.607155:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1240 00000100:00100000:3.0:1767319153.607163:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241 00000100:00100000:3.0:1767319153.607167:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982744576:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319153.607175:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319153.607195:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982744576:12345-0@lo:38:kworker.0 Request processed in 26us (313us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319153.607200:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241 00000100:00080000:3.0:1767319153.607231:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319153.607236:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319153.607239:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982744576:0@lo:38:kworker.0 00000100:00080000:3.0:1767319153.607258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319153.607262:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319153.607265:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982743808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319153.608168:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744192 00000100:00100000:2.0:1767319153.608177:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744192 00000100:00100000:2.0:1767319153.608221:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 335 00000100:00100000:2.0:1767319153.608227:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982744192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319153.608242:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319153.608315:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982744192:12345-0@lo:38:kworker.0 Request processed in 87us (1483us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319153.608321:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 335 00000100:00080000:3.0:1767319153.608749:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319153.608754:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319153.608758:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982744064:0@lo:38:kworker.0 00000100:00080000:3.0:1767319153.608767:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319153.608770:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319153.608773:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982744192:0@lo:38:kworker.0 00010000:00010000:1.0:1767319153.796724:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319154.820429:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319155.597885:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319155.597934:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319155.598068:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609088 00000100:00100000:2.0:1767319155.598079:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609088 00000100:00100000:2.0:1767319155.598157:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915608960 00000100:00100000:2.0:1767319155.598161:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915608960 00000100:00100000:2.0:1767319155.598185:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 946 00000100:00100000:2.0:1767319155.598195:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915609088:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319155.598319:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915609088:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 125us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319155.598326:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 946 00000100:00100000:2.0:1767319155.598345:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1242 00000100:00100000:2.0:1767319155.598350:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915608960:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319155.598369:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319155.598395:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915608960:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (460us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319155.598401:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1242 00000100:00080000:1.0:1767319155.600744:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319155.846103:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319156.868650:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319157.189420:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319157.189445:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982744704:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319157.189520:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319157.189713:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744704 00000100:00100000:1.0:1767319157.189721:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744704 00000100:00100000:1.0:1767319157.189795:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 648 00000100:00100000:1.0:1767319157.189806:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982744704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319157.189837:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319157.189881:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982744704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (360us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319157.189888:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 648 00000100:00080000:3.0:1767319157.189984:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319157.190021:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982744704:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319157.197818:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982744832:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319157.197888:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319157.197980:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744832 00000100:00100000:1.0:1767319157.197988:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744832 00000100:00080000:2.0:1767319157.198040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319157.198057:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 649 00000100:00100000:1.0:1767319157.198068:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982744832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319157.198106:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319157.203258:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982744832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5180us (5357us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319157.203289:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 649 00000100:00100000:3.0:1767319157.206991:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982744832:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319157.892432:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319158.724464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319158.724476:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319158.724562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319158.724566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319158.724570:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319158.724578:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4402 00000100:00080000:2.0:1767319158.724583:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319158.724589:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319158.724612:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962a00 x1853163982745088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319158.724644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319158.724647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319158.724649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319158.724654:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4402 00000100:00080000:2.0:1767319158.724656:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319158.724660:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319158.724670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962d80 x1853163982745344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319158.724678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319158.724681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319158.724682:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319158.724685:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4402 00000100:00080000:2.0:1767319158.724688:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319158.724691:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319158.724701:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961500 x1853163982745472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319158.724723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319158.724726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319158.724729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319158.724732:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4402 00000100:00080000:2.0:1767319158.724735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319158.724739:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319158.724749:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960a80 x1853163982745856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319158.724775:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982745088:0@lo:38:kworker.0 00000100:00100000:2.0:1767319158.724780:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982745216:0@lo:400:kworker.0 00000100:00100000:0.0:1767319158.724863:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982744960:0@lo:400:kworker.0 00000100:00100000:1.0:1767319158.724947:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982745728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319158.733751:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319158.733777:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982745600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319158.733835:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982745344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319158.739673:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319158.739731:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982745472:0@lo:38:kworker.0 00000100:00100000:3.0:1767319158.739780:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319158.739806:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982745856:0@lo:38:kworker.0 00000100:00100000:3.0:1767319158.739828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319158.739894:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745088 00000100:00100000:3.0:1767319158.739903:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745088 00000100:00100000:3.0:1767319158.739935:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745344 00000100:00100000:3.0:1767319158.739937:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745344 00000100:00100000:3.0:1767319158.739948:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745856 00000100:00100000:3.0:1767319158.739950:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745856 00000100:00100000:3.0:1767319158.739962:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1243 00000100:00100000:3.0:1767319158.739969:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319158.739992:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319158.740036:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745088:12345-0@lo:38:kworker.0 Request processed in 65us (6296us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319158.740042:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1243 00000100:00100000:3.0:1767319158.740060:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244 00000100:00100000:3.0:1767319158.740063:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319158.740073:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319158.740096:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745344:12345-0@lo:38:kworker.0 Request processed in 31us (431us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319158.740101:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244 00000100:00100000:3.0:1767319158.740113:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245 00000100:00100000:3.0:1767319158.740116:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745856:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319158.740124:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319158.740143:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982745856:12345-0@lo:38:kworker.0 Request processed in 25us (314us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319158.740150:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245 00000100:00100000:3.0:1767319158.740174:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745472 00000100:00100000:3.0:1767319158.740177:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745472 00000100:00100000:3.0:1767319158.740194:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 336 00000100:00100000:3.0:1767319158.740199:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d84850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982745472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319158.740208:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319158.740231:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d84850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982745472:12345-0@lo:38:kworker.0 Request processed in 31us (450us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319158.740237:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 336 00000100:00100000:0.0:1767319158.741345:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319158.741427:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745600 00000100:00100000:0.0:1767319158.741433:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745600 00000100:00100000:0.0:1767319158.741478:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 609 00000100:00100000:0.0:1767319158.741486:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982745600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319158.741544:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982745600:12345-0@lo:400:kworker.0 Request processed in 59us (203us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319158.741552:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 609 00000100:00100000:0.0:1767319158.741579:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319158.741608:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982744960 00000100:00100000:0.0:1767319158.741612:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982744960 00000100:00100000:0.0:1767319158.741645:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947 00000100:00100000:0.0:1767319158.741651:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982744960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319158.741684:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982744960:12345-0@lo:400:kworker.0 Request processed in 33us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319158.741690:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947 00000100:00080000:0.0:1767319158.741706:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319158.741743:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f961f80 x1853163982744960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319174 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319158.741758:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982744960:0@lo:400:kworker.0 00000100:00100000:0.0:1767319158.741794:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982745600:0@lo:400:kworker.0 00000100:00100000:2.0:1767319158.742029:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319158.742078:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745216 00000100:00100000:2.0:1767319158.742083:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745216 00000100:00100000:2.0:1767319158.742113:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948 00000100:00100000:2.0:1767319158.742120:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982745216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319158.742157:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982745216:12345-0@lo:400:kworker.0 Request processed in 38us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319158.742163:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948 00010000:00080000:2.0:1767319158.742197:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963b80 x1853163982745216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319174 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319158.742207:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982745216:0@lo:400:kworker.0 00000100:00080000:2.0:1767319158.742459:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319158.742465:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319158.742469:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982745088:0@lo:38:kworker.0 00000100:00080000:2.0:1767319158.742481:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319158.742485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319158.742487:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982745344:0@lo:38:kworker.0 00000100:00080000:2.0:1767319158.742497:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319158.742500:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319158.742503:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982745472:0@lo:38:kworker.0 00000100:00080000:2.0:1767319158.742514:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319158.742517:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319158.742519:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982745856:0@lo:38:kworker.0 00000100:00080000:0.0:1767319158.742622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319158.742733:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319158.742967:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745728 00000100:00100000:0.0:1767319158.742973:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745728 00000100:00100000:0.0:1767319158.743010:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 610 00000100:00100000:0.0:1767319158.743017:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982745728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319158.743071:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982745728:12345-0@lo:400:kworker.0 Request processed in 54us (338us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319158.743078:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 610 00000100:00100000:2.0:1767319158.743185:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982745728:0@lo:400:kworker.0 00010000:00010000:1.0:1767319158.916484:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319159.945370:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319160.717853:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319160.717907:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319160.717931:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915609472 offset 0 length 520: 4 00000100:00100000:2.0:1767319160.717974:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609344 00000100:00100000:2.0:1767319160.717985:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609344 00000100:00100000:2.0:1767319160.718053:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609216 00000100:00100000:2.0:1767319160.718057:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609216 00000100:00100000:2.0:1767319160.718077:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 949 00000100:00100000:2.0:1767319160.718086:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915609344:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319160.718130:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319160.718160:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915609344:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 73us (325us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319160.718168:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1246 00000100:00100000:2.0:1767319160.718179:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 949 00000100:00100000:3.0:1767319160.718181:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915609216:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319160.718204:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319160.718237:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915609216:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (329us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319160.718243:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1246 00010000:00010000:0.0:1767319160.964594:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319161.988496:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319162.310844:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982745984:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319162.310878:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319162.310892:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982746112:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319162.310912:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319162.310979:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982745984 00000100:00100000:1.0:1767319162.310987:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982745984 00000100:00100000:1.0:1767319162.311038:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 650 00000100:00100000:1.0:1767319162.311046:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982745984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319162.311073:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767319162.311128:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319162.318567:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319162.318830:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746112 00000100:00100000:3.0:1767319162.318840:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746112 00000100:00100000:3.0:1767319162.318922:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 651 00000100:00100000:3.0:1767319162.318933:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982746112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319162.318969:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319162.319030:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982746112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (470us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319162.319037:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 651 00000100:00100000:3.0:1767319162.319088:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982746112:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319162.320480:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319162.327117:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982745984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 16062us (16194us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319162.327132:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 650 00000100:00100000:0.0:1767319162.329587:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982745984:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319163.012482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319163.844433:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319138/real 1767319138] req@ffff9346382df800 x1853163982739712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319163 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319163.844467:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346382df800 x1853163982739712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319163 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319163.844486:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319163.844495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319163.844498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982739712:0@lo:250:kworker.0 00000100:00080000:0.0:1767319163.844656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319163.844662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319163.844665:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319163.844670:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4386 00000100:00080000:0.0:1767319163.844673:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319163.844679:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319163.844711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5180 x1853163982746240/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319163.844767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319163.844771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319163.844772:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319163.844776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4407 00000100:00080000:0.0:1767319163.844778:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319163.844781:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319163.844828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7800 x1853163982746496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319163.844862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319163.844866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319163.844868:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319163.844871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4407 00000100:00080000:0.0:1767319163.844874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319163.844877:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319163.844892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982746752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319163.844900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319163.844903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319163.844905:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319163.844907:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4407 00000100:00080000:0.0:1767319163.844909:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319163.844912:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319163.844922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163982746880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319163.844940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319163.844943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319163.844944:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319163.844947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4407 00000100:00080000:0.0:1767319163.844950:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319163.844959:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319163.844971:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163982747264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319163.845002:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982746624:0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.845010:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982746368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319163.845091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982747136:0@lo:400:kworker.0 00000100:00100000:2.0:1767319163.845450:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982746240:0@lo:250:kworker.0 00000100:00100000:3.0:1767319163.846494:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319163.846606:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982747008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.846642:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319163.846682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746368 00000100:00100000:3.0:1767319163.846689:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746368 00000100:00100000:3.0:1767319163.846744:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 950 00000100:00100000:3.0:1767319163.846754:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982746368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.847159:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982746368:12345-0@lo:400:kworker.0 Request processed in 405us (668us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319163.847169:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 950 00010000:00080000:3.0:1767319163.847216:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982746368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319179 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319163.847228:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982746368:0@lo:400:kworker.0 00000100:00080000:3.0:1767319163.847373:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319163.847395:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747008 00000100:00100000:3.0:1767319163.847400:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747008 00000100:00100000:3.0:1767319163.847431:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 611 00000100:00100000:3.0:1767319163.847437:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982747008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.847479:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982747008:12345-0@lo:400:kworker.0 Request processed in 42us (836us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319163.847486:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 611 00000100:00100000:3.0:1767319163.847611:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982747008:0@lo:400:kworker.0 00000100:00100000:1.0:1767319163.847672:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319163.847820:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747136 00000100:00100000:3.0:1767319163.847824:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747136 00000100:00100000:3.0:1767319163.847857:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 612 00000100:00100000:3.0:1767319163.847865:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982747136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.847901:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982747136:12345-0@lo:400:kworker.0 Request processed in 37us (230us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319163.847908:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 612 00000100:00100000:1.0:1767319163.848162:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982747136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319163.849581:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319163.849723:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746624 00000100:00100000:3.0:1767319163.849730:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746624 00000100:00100000:3.0:1767319163.849811:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951 00000100:00100000:3.0:1767319163.849820:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982746624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319163.849881:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982746624:12345-0@lo:400:kworker.0 Request processed in 62us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319163.849888:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951 00000100:00080000:0.0:1767319163.850057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319163.850087:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6d80 x1853163982746624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319179 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319163.850100:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982746624:0@lo:400:kworker.0 00000400:00000100:2.0:1767319163.850243:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982746240 offset 0 length 520: 4 00000100:00100000:2.0:1767319163.850299:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982746496:0@lo:38:kworker.0 00000100:00100000:2.0:1767319163.850334:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319163.850469:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982746752:0@lo:38:kworker.0 00000100:00100000:2.0:1767319163.850491:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319163.851682:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982746880:0@lo:38:kworker.0 00000100:00100000:2.0:1767319163.851768:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319163.851807:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982747264:0@lo:38:kworker.0 00000100:00100000:2.0:1767319163.851829:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319163.851961:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746880 00000100:00100000:0.0:1767319163.851970:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746880 00000100:00100000:0.0:1767319163.852007:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 337 00000100:00100000:0.0:1767319163.852014:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982746880:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319163.852034:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319163.852074:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982746880:12345-0@lo:38:kworker.0 Request processed in 59us (305us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319163.852081:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 337 00000100:00100000:1.0:1767319163.853587:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746496 00000100:00100000:1.0:1767319163.853602:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746496 00000100:00100000:1.0:1767319163.853653:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982746752 00000100:00100000:1.0:1767319163.853657:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982746752 00000100:00100000:1.0:1767319163.853680:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247 00000100:00100000:1.0:1767319163.853688:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982746496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319163.853708:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319163.853767:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982746496:12345-0@lo:38:kworker.0 Request processed in 79us (3432us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319163.853773:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247 00000100:00100000:1.0:1767319163.853801:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248 00000100:00100000:1.0:1767319163.853805:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982746752:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319163.853813:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319163.853838:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982746752:12345-0@lo:38:kworker.0 Request processed in 32us (3346us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319163.853844:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248 00000100:00080000:2.0:1767319163.857443:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319163.857456:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319163.857462:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982746496:0@lo:38:kworker.0 00000100:00080000:2.0:1767319163.857475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319163.857479:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319163.857481:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982746752:0@lo:38:kworker.0 00000100:00080000:2.0:1767319163.857497:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319163.857501:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319163.857504:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982746880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319163.857674:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747264 00000100:00100000:0.0:1767319163.857682:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747264 00000100:00100000:0.0:1767319163.857732:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1249 00000100:00100000:0.0:1767319163.857740:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982747264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319163.857760:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319163.857838:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982747264:12345-0@lo:38:kworker.0 Request processed in 98us (6007us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319163.857846:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1249 00000100:00080000:2.0:1767319163.858135:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319163.858140:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319163.858143:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982747264:0@lo:38:kworker.0 00010000:00010000:0.0:1767319164.036433:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319165.060810:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319165.835882:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319165.836470:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609728 00000100:00100000:3.0:1767319165.836481:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609728 00000100:00100000:3.0:1767319165.836565:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 952 00000100:00100000:3.0:1767319165.836575:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915609728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319165.836662:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915609728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (791us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319165.836670:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 952 00000100:00080000:0.0:1767319165.836799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319165.848173:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319165.851220:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609600 00000100:00100000:2.0:1767319165.851235:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609600 00000100:00100000:2.0:1767319165.851333:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1250 00000100:00100000:2.0:1767319165.851342:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915609600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319165.851368:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319165.851451:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915609600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 108us (3291us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319165.851462:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1250 00010000:00010000:1.0:1767319166.084472:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319167.109224:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319167.433165:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319167.433510:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982747520:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319167.433584:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319167.433665:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747520 00000100:00100000:1.0:1767319167.433675:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747520 00000100:00100000:1.0:1767319167.433747:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 652 00000100:00100000:1.0:1767319167.433772:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982747520:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319167.433820:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767319167.433883:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319167.436016:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982747392:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319167.436084:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319167.436217:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747392 00000100:00100000:2.0:1767319167.436225:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747392 00000100:00100000:2.0:1767319167.436336:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 653 00000100:00100000:2.0:1767319167.436346:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982747392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319167.436375:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319167.436615:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982747392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 270us (529us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319167.436623:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 653 00000100:00080000:0.0:1767319167.437703:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319167.437944:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982747392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319167.438189:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982747520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4431us (4602us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319167.438199:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 652 00000100:00100000:3.0:1767319167.439787:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982747520:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319168.132650:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319168.964863:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319168.964873:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319168.964927:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319168.964931:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319168.964935:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319168.964943:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4412 00000100:00080000:3.0:1767319168.964947:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319168.964953:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319168.964973:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911880 x1853163982747776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319168.965039:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982747776:0@lo:38:kworker.0 00000100:00100000:1.0:1767319168.965104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319168.965142:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982747648:0@lo:400:kworker.0 00000100:00080000:3.0:1767319168.965177:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319168.965184:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319168.965185:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319168.965189:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319168.965193:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4412 00000100:00080000:3.0:1767319168.965196:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319168.965200:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319168.965214:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910e00 x1853163982748032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319168.965229:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747776 00000100:00100000:2.0:1767319168.965237:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747776 00000100:00100000:2.0:1767319168.965289:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747648 00000100:00100000:2.0:1767319168.965293:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747648 00000100:00100000:0.0:1767319168.965306:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251 00000100:00100000:0.0:1767319168.965316:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982747776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319168.965337:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319168.965338:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953 00000100:00100000:2.0:1767319168.965348:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982747648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319168.965397:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982747648:12345-0@lo:400:kworker.0 Request processed in 49us (216us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319168.965404:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953 00010000:00080000:2.0:1767319168.965439:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a913480 x1853163982747648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319184 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319168.965451:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982747648:0@lo:400:kworker.0 00000100:00100000:2.0:1767319168.965495:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319168.965502:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982747904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319168.965529:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319168.965552:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982747904 00000100:00100000:2.0:1767319168.965556:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982747904 00000100:00100000:2.0:1767319168.965580:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 954 00000100:00100000:2.0:1767319168.965586:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982747904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319168.965619:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982747904:12345-0@lo:400:kworker.0 Request processed in 33us (89us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319168.965625:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 954 00010000:00080000:2.0:1767319168.965646:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a913800 x1853163982747904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319184 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319168.965740:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982747904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319168.965746:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982748032:0@lo:38:kworker.0 00000100:00100000:1.0:1767319168.965776:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319168.965777:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319168.965803:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748032 00000100:00080000:3.0:1767319168.965832:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319168.965837:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319168.965839:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319168.965842:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4412 00000100:00080000:3.0:1767319168.965845:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319168.965849:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319168.965865:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163982748160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319168.965880:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748032 00000100:00100000:2.0:1767319168.965907:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1252 00000100:00080000:3.0:1767319168.965909:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319168.965914:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982748032:12345-0@lo:38:kworker.0 00000100:00080000:3.0:1767319168.965917:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319168.965922:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319168.965928:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4412 00010000:00020000:2.0:1767319168.965929:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319168.965933:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319168.965938:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319168.965952:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163982748544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319168.965961:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982748032:12345-0@lo:38:kworker.0 Request processed in 46us (184us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319168.965970:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1252 00000100:00100000:3.0:1767319168.965982:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982748416:0@lo:400:kworker.0 00000100:00100000:2.0:1767319168.965988:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319168.965997:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982748288:0@lo:400:kworker.0 00000100:00100000:3.0:1767319168.966026:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319168.966063:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982747776:12345-0@lo:38:kworker.0 Request processed in 748us (959us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319168.966070:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251 00000100:00100000:3.0:1767319168.966423:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748416 00000100:00100000:3.0:1767319168.966428:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748416 00000100:00100000:3.0:1767319168.966457:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 613 00000100:00100000:3.0:1767319168.966463:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982748416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319168.966501:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982748416:12345-0@lo:400:kworker.0 Request processed in 38us (477us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319168.966508:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 613 00000100:00100000:3.0:1767319168.966539:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982748416:0@lo:400:kworker.0 00000100:00100000:0.0:1767319168.966618:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319168.966764:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748288 00000100:00100000:3.0:1767319168.966769:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748288 00000100:00100000:3.0:1767319168.966800:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 614 00000100:00100000:3.0:1767319168.966807:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982748288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319168.966841:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982748288:12345-0@lo:400:kworker.0 Request processed in 34us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319168.966847:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 614 00000100:00100000:0.0:1767319168.966915:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982748288:0@lo:400:kworker.0 00000100:00080000:1.0:1767319168.969326:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319168.969339:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319168.969344:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982747776:0@lo:38:kworker.0 00000100:00100000:1.0:1767319168.969378:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982748160:0@lo:38:kworker.0 00000100:00100000:1.0:1767319168.969437:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319168.969482:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982748544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319168.969506:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319168.969540:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319168.969544:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319168.969548:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982748032:0@lo:38:kworker.0 00000100:00100000:0.0:1767319168.969826:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748160 00000100:00100000:0.0:1767319168.969833:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748160 00000100:00100000:0.0:1767319168.969879:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 338 00000100:00100000:0.0:1767319168.969886:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982748160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319168.969905:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319168.969959:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982748160:12345-0@lo:38:kworker.0 Request processed in 72us (521us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319168.969966:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 338 00000100:00080000:1.0:1767319168.970033:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319168.970037:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319168.970039:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982748160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319168.970163:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748544 00000100:00100000:2.0:1767319168.970168:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748544 00000100:00100000:2.0:1767319168.970258:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253 00000100:00100000:2.0:1767319168.970264:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982748544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319168.970295:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319168.970339:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982748544:12345-0@lo:38:kworker.0 Request processed in 74us (831us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319168.970348:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253 00000100:00080000:1.0:1767319168.970363:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319168.970468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319168.970473:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982748544:0@lo:38:kworker.0 00010000:00010000:1.0:1767319169.156595:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319170.180420:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319170.958557:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319170.958744:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319170.959352:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609984 00000100:00100000:0.0:1767319170.959364:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609984 00000100:00100000:0.0:1767319170.959433:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915609856 00000100:00100000:0.0:1767319170.959436:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915609856 00000100:00100000:0.0:1767319170.959458:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 955 00000100:00100000:0.0:1767319170.959469:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915609984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319170.959546:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915609984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (1001us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319170.959554:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 955 00000100:00100000:0.0:1767319170.959572:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1254 00000100:00100000:0.0:1767319170.959576:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915609856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319170.959595:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319170.959620:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915609856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (877us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319170.959626:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1254 00000100:00080000:2.0:1767319170.959894:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319171.209470:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319172.228542:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319172.549433:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982748672:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319172.549547:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319172.550163:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748672 00000100:00100000:1.0:1767319172.550172:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748672 00000100:00100000:1.0:1767319172.550234:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 654 00000100:00100000:1.0:1767319172.550243:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982748672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319172.550309:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319172.550364:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982748672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 121us (817us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319172.550374:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 654 00000100:00100000:1.0:1767319172.550432:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982748672:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319172.550509:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982748800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319172.550541:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319172.550641:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319172.553784:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748800 00000100:00100000:1.0:1767319172.553801:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748800 00000100:00100000:1.0:1767319172.553890:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767319172.553902:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982748800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319172.553939:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319172.554030:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982748800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 131us (3488us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319172.554037:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767319172.554091:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982748800:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319172.554243:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:0.0:1767319173.252522:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319174.084496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319174.084508:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319174.084556:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319174.084560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319174.084563:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319174.084571:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4417 00000100:00080000:2.0:1767319174.084576:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319174.084582:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319174.084612:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ffb80 x1853163982749056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319174.084669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319174.084673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319174.084674:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319174.084678:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4417 00000100:00080000:2.0:1767319174.084681:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319174.084684:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319174.084696:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fed80 x1853163982749312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319174.084706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319174.084709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319174.084711:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319174.084714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4417 00000100:00080000:2.0:1767319174.084717:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319174.084720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319174.084731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163982749440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319174.084874:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982749184:0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.084942:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319174.084981:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319174.084987:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982749568:0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085013:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319174.085046:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749184 00000100:00100000:0.0:1767319174.085053:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749184 00000100:00100000:0.0:1767319174.085106:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956 00000100:00100000:0.0:1767319174.085116:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982749184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085162:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982749184:12345-0@lo:400:kworker.0 Request processed in 48us (220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319174.085169:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956 00000100:00100000:0.0:1767319174.085192:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749568 00000100:00100000:0.0:1767319174.085196:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749568 00000100:00100000:0.0:1767319174.085237:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 615 00000100:00100000:0.0:1767319174.085244:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982749568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085300:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982749568:12345-0@lo:400:kworker.0 Request processed in 56us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319174.085306:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 615 00000100:00080000:0.0:1767319174.085321:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319174.085325:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319174.085344:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982748928:0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085369:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319174.085391:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982748928 00000100:00100000:0.0:1767319174.085394:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982748928 00000100:00100000:0.0:1767319174.085415:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957 00000100:00100000:0.0:1767319174.085419:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982748928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085446:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982748928:12345-0@lo:400:kworker.0 Request processed in 27us (77us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319174.085452:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957 00000100:00080000:0.0:1767319174.085463:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319174.085481:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fca80 x1853163982748928/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319190 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319174.085492:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982748928:0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.085516:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982749568:0@lo:400:kworker.0 00010000:00080000:0.0:1767319174.085525:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff100 x1853163982749184/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319190 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319174.085533:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982749184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319174.085771:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982749056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319174.085800:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.085826:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982749312:0@lo:38:kworker.0 00000100:00100000:3.0:1767319174.085853:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.085869:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982749440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319174.085889:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.085928:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749440 00000100:00100000:3.0:1767319174.085932:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749440 00000100:00100000:3.0:1767319174.085953:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 339 00000100:00100000:3.0:1767319174.085958:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982749440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319174.085975:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319174.086005:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982749440:12345-0@lo:38:kworker.0 Request processed in 45us (114us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319174.086012:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 339 00000100:00100000:3.0:1767319174.086031:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749056 00000100:00100000:3.0:1767319174.086034:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749056 00000100:00100000:3.0:1767319174.086048:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749312 00000100:00100000:3.0:1767319174.086050:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749312 00000100:00100000:3.0:1767319174.086063:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1255 00000100:00100000:3.0:1767319174.086067:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982749056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319174.086076:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319174.086098:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982749056:12345-0@lo:38:kworker.0 Request processed in 30us (297us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319174.086103:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1255 00000100:00100000:3.0:1767319174.086114:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256 00000100:00100000:3.0:1767319174.086118:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982749312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319174.086126:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319174.086148:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982749312:12345-0@lo:38:kworker.0 Request processed in 28us (293us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319174.086153:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256 00000100:00080000:3.0:1767319174.086180:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319174.086185:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319174.086188:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982749056:0@lo:38:kworker.0 00000100:00080000:3.0:1767319174.086196:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319174.086199:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319174.086201:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982749312:0@lo:38:kworker.0 00000100:00080000:3.0:1767319174.086208:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319174.086212:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319174.086214:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982749440:0@lo:38:kworker.0 00000100:00080000:2.0:1767319174.086879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319174.086884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319174.086887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319174.086891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4417 00000100:00080000:2.0:1767319174.086913:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319174.086919:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319174.086941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fea00 x1853163982749824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319174.087094:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982749696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319174.087134:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319174.087537:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982749824:0@lo:38:kworker.0 00000100:00100000:3.0:1767319174.087564:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319174.088375:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749696 00000100:00100000:2.0:1767319174.088385:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749696 00000100:00100000:2.0:1767319174.088456:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 616 00000100:00100000:2.0:1767319174.088464:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982749696:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319174.088524:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982749696:12345-0@lo:400:kworker.0 Request processed in 60us (1389us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319174.088531:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 616 00000100:00100000:1.0:1767319174.088652:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982749696:0@lo:400:kworker.0 00000100:00100000:0.0:1767319174.089045:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749824 00000100:00100000:0.0:1767319174.089051:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749824 00000100:00100000:0.0:1767319174.089086:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257 00000100:00100000:0.0:1767319174.089092:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982749824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319174.089111:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319174.089164:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982749824:12345-0@lo:38:kworker.0 Request processed in 71us (1599us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319174.089170:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257 00000100:00080000:3.0:1767319174.089252:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319174.089908:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319174.089915:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319174.089919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982749824:0@lo:38:kworker.0 00010000:00010000:1.0:1767319174.276522:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319175.302198:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319176.076617:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319176.076712:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610112 00000100:00100000:2.0:1767319176.076723:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610112 00000100:00100000:2.0:1767319176.076773:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1258 00000100:00100000:2.0:1767319176.076783:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915610112:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319176.076809:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319176.076885:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915610112:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (278us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319176.076893:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1258 00000100:00100000:3.0:1767319176.079696:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319176.079803:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610240 00000100:00100000:0.0:1767319176.079815:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610240 00000100:00100000:0.0:1767319176.079893:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 958 00000100:00100000:0.0:1767319176.079903:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915610240:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319176.080001:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915610240:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319176.080009:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 958 00000100:00080000:3.0:1767319176.080244:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319176.324433:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319177.348901:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319177.671340:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982749952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319177.671413:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319177.671661:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982749952 00000100:00100000:1.0:1767319177.671669:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982749952 00000100:00100000:1.0:1767319177.671733:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767319177.671742:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982749952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319177.671775:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319177.671822:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982749952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319177.671830:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 656 00000100:00080000:0.0:1767319177.672035:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319177.672071:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982749952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319177.672815:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982750080:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319177.672868:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319177.673150:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750080 00000100:00100000:2.0:1767319177.673157:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750080 00000100:00100000:2.0:1767319177.673211:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 657 00000100:00100000:2.0:1767319177.673220:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982750080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319177.673244:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319177.673303:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982750080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (435us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319177.673310:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 657 00000100:00100000:2.0:1767319177.673344:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982750080:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319177.673558:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319178.372584:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319179.204420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319179.204433:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319179.204507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319179.204511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319179.204515:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319179.204525:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4422 00000100:00080000:2.0:1767319179.204530:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319179.204537:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319179.204565:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163982750336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319179.204614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319179.204618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319179.204619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319179.204623:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4422 00000100:00080000:2.0:1767319179.204626:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319179.204630:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319179.204652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdc00 x1853163982750592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319179.204662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319179.204664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319179.204666:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319179.204670:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4422 00000100:00080000:2.0:1767319179.204672:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319179.204676:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319179.204688:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163982750720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319179.204711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319179.204714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319179.204716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319179.204720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4422 00000100:00080000:2.0:1767319179.204722:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319179.204725:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319179.204735:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fce00 x1853163982751104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319179.204770:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982750464:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.204840:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319179.204844:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982750336:0@lo:38:kworker.0 00000100:00100000:3.0:1767319179.204872:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319179.204896:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982750592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319179.204899:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319179.204904:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982750848:0@lo:400:kworker.0 00000100:00100000:3.0:1767319179.204916:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319179.204926:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319179.204938:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982750720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319179.204962:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319179.204978:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750464 00000100:00100000:3.0:1767319179.204984:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982751104:0@lo:38:kworker.0 00000100:00100000:2.0:1767319179.204986:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750464 00000100:00100000:3.0:1767319179.205003:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319179.205028:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750336 00000100:00100000:3.0:1767319179.205032:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750336 00000100:00100000:2.0:1767319179.205036:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750592 00000100:00100000:2.0:1767319179.205039:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750592 00000100:00100000:3.0:1767319179.205043:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751104 00000100:00100000:3.0:1767319179.205045:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751104 00000100:00100000:2.0:1767319179.205051:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959 00000100:00100000:2.0:1767319179.205059:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982750464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.205102:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982750976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.205112:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982750464:12345-0@lo:400:kworker.0 Request processed in 55us (272us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319179.205127:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.205147:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319179.205152:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982750208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.205171:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319179.205190:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750208 00000100:00100000:0.0:1767319179.205193:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750208 00000100:00100000:3.0:1767319179.205232:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259 00000100:00100000:3.0:1767319179.205237:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982750336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319179.205254:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319179.205322:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982750336:12345-0@lo:38:kworker.0 Request processed in 83us (449us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319179.205328:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259 00000100:00100000:3.0:1767319179.205344:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260 00000100:00100000:3.0:1767319179.205347:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982750592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319179.205357:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319179.205376:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982750592:12345-0@lo:38:kworker.0 Request processed in 28us (459us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319179.205381:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260 00000100:00100000:3.0:1767319179.205392:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1261 00000100:00100000:3.0:1767319179.205396:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982751104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319179.205404:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319179.205425:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982751104:12345-0@lo:38:kworker.0 Request processed in 28us (421us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319179.205429:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1261 00000100:00100000:3.0:1767319179.205446:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750720 00000100:00100000:3.0:1767319179.205449:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750720 00000100:00100000:3.0:1767319179.205466:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 340 00000100:00100000:3.0:1767319179.205470:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982750720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319179.205478:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319179.205514:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982750720:12345-0@lo:38:kworker.0 Request processed in 42us (551us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319179.205519:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 340 00000100:00080000:3.0:1767319179.205534:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319179.205564:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319179.205569:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319179.205574:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982750336:0@lo:38:kworker.0 00000100:00080000:3.0:1767319179.205582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319179.205585:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319179.205587:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982750592:0@lo:38:kworker.0 00000100:00080000:3.0:1767319179.205611:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319179.205614:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319179.205616:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750848 00000100:00100000:3.0:1767319179.205617:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982750720:0@lo:38:kworker.0 00000100:00100000:1.0:1767319179.205619:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750848 00000100:00080000:3.0:1767319179.205624:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319179.205627:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319179.205629:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982751104:0@lo:38:kworker.0 00000100:00100000:1.0:1767319179.205663:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982750976 00000100:00100000:1.0:1767319179.205667:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982750976 00000100:00100000:1.0:1767319179.205685:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 617 00000100:00100000:1.0:1767319179.205690:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982750848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.205725:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982750848:12345-0@lo:400:kworker.0 Request processed in 35us (798us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319179.205731:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 617 00000100:00100000:1.0:1767319179.205744:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 618 00000100:00100000:1.0:1767319179.205750:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982750976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319179.205755:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982750848:0@lo:400:kworker.0 00010000:00080000:3.0:1767319179.205766:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc380 x1853163982750464/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319195 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319179.205778:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982750976:12345-0@lo:400:kworker.0 Request processed in 29us (651us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319179.205781:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982750464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319179.205784:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 618 00000100:00080000:1.0:1767319179.205802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319179.205805:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319179.205822:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982750976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319179.206013:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959 00000100:00100000:0.0:1767319179.209180:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960 00000100:00100000:0.0:1767319179.209205:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982750208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319179.209337:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982750208:12345-0@lo:400:kworker.0 Request processed in 145us (4164us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319179.209345:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960 00010000:00080000:3.0:1767319179.209577:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd500 x1853163982750208/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319195 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319179.209750:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982750208:0@lo:400:kworker.0 00010000:00010000:0.0:1767319179.396889:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319180.420694:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319181.200490:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319181.200543:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319181.200763:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610496 00000100:00100000:1.0:1767319181.200774:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610496 00000100:00100000:1.0:1767319181.200851:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610368 00000100:00100000:1.0:1767319181.200857:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610368 00000100:00100000:1.0:1767319181.200880:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 961 00000100:00100000:1.0:1767319181.200889:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915610496:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319181.200986:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915610496:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (507us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319181.200994:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 961 00000100:00100000:1.0:1767319181.201013:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1262 00000100:00100000:1.0:1767319181.201018:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915610368:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319181.201038:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319181.201065:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915610368:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (522us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319181.201070:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1262 00000100:00080000:3.0:1767319181.201663:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319181.445396:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319182.468453:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319182.789304:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319182.789327:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982751232:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319182.789398:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319182.789526:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751232 00000100:00100000:1.0:1767319182.789535:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751232 00000100:00100000:1.0:1767319182.789598:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 658 00000100:00100000:1.0:1767319182.789608:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982751232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319182.789638:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319182.789689:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982751232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (290us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319182.789696:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 658 00000100:00100000:1.0:1767319182.789742:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982751232:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319182.789799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319182.790174:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319182.790184:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982751360:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319182.790217:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319182.790513:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751360 00000100:00100000:2.0:1767319182.790518:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751360 00000100:00100000:2.0:1767319182.790568:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 659 00000100:00100000:2.0:1767319182.790576:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982751360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319182.790595:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319182.790642:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982751360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (424us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319182.790649:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 659 00000100:00100000:0.0:1767319182.790721:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982751360:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319182.790771:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319183.496847:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319184.326719:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319184.326730:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319184.326800:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319184.326805:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319184.326810:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319184.326819:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4427 00000100:00080000:2.0:1767319184.326824:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319184.326832:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319184.326855:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fed80 x1853163982751616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319184.327041:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982751488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.327123:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319184.327304:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319184.327309:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319184.327311:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319184.327315:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4427 00000100:00080000:2.0:1767319184.327318:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319184.327322:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319184.327337:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163982751872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319184.327346:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319184.327349:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319184.327351:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319184.327354:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4427 00000100:00080000:2.0:1767319184.327356:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319184.327359:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319184.327369:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe300 x1853163982752000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319184.327395:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319184.327399:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319184.327410:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319184.327414:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4427 00000100:00080000:2.0:1767319184.327416:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319184.327420:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319184.327433:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd500 x1853163982752384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319184.327455:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319184.327463:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982752256:0@lo:400:kworker.0 00000100:00100000:2.0:1767319184.327496:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.327739:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982752128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.327767:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.327796:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319184.327802:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982751744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.327825:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.327855:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751488 00000100:00100000:0.0:1767319184.327861:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751488 00000100:00100000:0.0:1767319184.327918:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751744 00000100:00100000:0.0:1767319184.327921:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751744 00000100:00100000:0.0:1767319184.327985:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 962 00000100:00100000:0.0:1767319184.327994:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982751488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.328047:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982751488:12345-0@lo:400:kworker.0 Request processed in 54us (923us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319184.328054:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 962 00000100:00100000:0.0:1767319184.328215:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963 00000100:00100000:0.0:1767319184.328223:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982751744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319184.328262:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982751744:12345-0@lo:400:kworker.0 Request processed in 40us (436us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319184.328305:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963 00010000:00080000:0.0:1767319184.328362:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ffb80 x1853163982751744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319200 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319184.328376:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982751744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319184.328486:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752256 00000100:00100000:3.0:1767319184.328491:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752256 00000100:00100000:3.0:1767319184.328530:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752128 00000100:00100000:3.0:1767319184.328534:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752128 00000100:00100000:3.0:1767319184.328555:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 619 00000100:00100000:3.0:1767319184.328561:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982752256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319184.328610:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982752256:12345-0@lo:400:kworker.0 Request processed in 50us (1113us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319184.328617:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 619 00000100:00100000:3.0:1767319184.328629:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 620 00000100:00100000:3.0:1767319184.328635:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982752128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319184.328669:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982752128:12345-0@lo:400:kworker.0 Request processed in 34us (901us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319184.328676:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 620 00000100:00080000:3.0:1767319184.328692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319184.328696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319184.328698:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319184.328727:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982752128:0@lo:400:kworker.0 00000100:00100000:2.0:1767319184.328873:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982752256:0@lo:400:kworker.0 00010000:00080000:2.0:1767319184.328884:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163982751488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319200 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319184.328892:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982751488:0@lo:400:kworker.0 00000100:00100000:1.0:1767319184.330571:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982751616:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.330641:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.330695:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982751872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.330721:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319184.330752:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982752000:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.330778:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319184.330797:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751616 00000100:00100000:3.0:1767319184.330811:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751616 00000100:00100000:1.0:1767319184.330820:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982752384:0@lo:38:kworker.0 00000100:00100000:1.0:1767319184.330847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319184.330913:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752000 00000100:00100000:0.0:1767319184.330918:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752000 00000100:00100000:0.0:1767319184.330949:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 341 00000100:00100000:0.0:1767319184.330957:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982752000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319184.330977:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319184.331020:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982751872 00000100:00100000:0.0:1767319184.331025:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982752000:12345-0@lo:38:kworker.0 Request processed in 66us (243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319184.331026:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982751872 00000100:00100000:0.0:1767319184.331035:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 341 00000100:00100000:0.0:1767319184.331096:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752384 00000100:00100000:0.0:1767319184.331101:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752384 00000100:00100000:0.0:1767319184.331115:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263 00000100:00100000:0.0:1767319184.331119:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982751616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319184.331130:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319184.331156:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982751616:12345-0@lo:38:kworker.0 Request processed in 35us (516us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319184.331161:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263 00000100:00100000:0.0:1767319184.331175:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1264 00000100:00100000:0.0:1767319184.331178:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982752384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319184.331187:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319184.331210:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982752384:12345-0@lo:38:kworker.0 Request processed in 30us (363us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319184.331216:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1264 00000100:00080000:0.0:1767319184.331263:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319184.331294:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319184.331299:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982751616:0@lo:38:kworker.0 00000100:00080000:0.0:1767319184.331311:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319184.331314:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319184.331317:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982752000:0@lo:38:kworker.0 00000100:00080000:0.0:1767319184.331325:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319184.331329:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319184.331331:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982752384:0@lo:38:kworker.0 00000100:00100000:3.0:1767319184.332464:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265 00000100:00100000:3.0:1767319184.332472:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982751872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319184.332486:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319184.332545:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982751872:12345-0@lo:38:kworker.0 Request processed in 73us (1823us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319184.332553:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265 00000100:00080000:3.0:1767319184.332591:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319184.332595:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319184.332598:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982751872:0@lo:38:kworker.0 00010000:00010000:1.0:1767319184.519762:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319185.540962:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319186.319859:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767319186.319897:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915610624 offset 0 length 520: 4 00000100:00100000:1.0:1767319186.319919:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319186.320013:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610880 00000100:00100000:2.0:1767319186.320023:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610880 00000100:00080000:0.0:1767319186.320113:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319186.320114:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915610752 00000100:00100000:2.0:1767319186.320120:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915610752 00000100:00100000:2.0:1767319186.320146:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 964 00000100:00100000:2.0:1767319186.320157:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915610880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319186.320162:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1266 00000100:00100000:0.0:1767319186.320172:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915610752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319186.320201:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319186.320500:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915610880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 342us (650us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319186.320511:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 964 00000100:00100000:0.0:1767319186.321162:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915610752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 985us (1237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319186.321171:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1266 00010000:00010000:1.0:1767319186.566067:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319187.588505:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319187.908585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982752640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319187.908586:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982752512:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319187.911906:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319187.912303:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752640 00000100:00100000:0.0:1767319187.912315:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752640 00000100:00100000:0.0:1767319187.912390:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 660 00000100:00100000:0.0:1767319187.912402:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982752640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319187.912434:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319187.912585:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982752640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 185us (686us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319187.912593:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 660 00000100:00080000:2.0:1767319187.914021:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319187.914265:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982752640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319187.914587:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319187.915130:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752512 00000100:00100000:2.0:1767319187.915139:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752512 00000100:00080000:0.0:1767319187.915211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319187.915211:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 661 00000100:00100000:2.0:1767319187.915222:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982752512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319187.915252:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319187.915348:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982752512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (761us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319187.915356:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 661 00000100:00100000:3.0:1767319187.915572:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982752512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319188.421695:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319163/real 1767319163] req@ffff9346059d5180 x1853163982746240/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319188 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319188.421744:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346059d5180 x1853163982746240/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319188 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319188.421766:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319188.421792:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319188.421797:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982746240:0@lo:250:kworker.0 00000100:00080000:2.0:1767319188.421951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319188.421957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319188.421960:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319188.421965:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4412 00000100:00080000:2.0:1767319188.421969:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319188.421976:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319188.422009:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982752768/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319188.422065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319188.422069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319188.422072:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319188.422075:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4432 00000100:00080000:2.0:1767319188.422078:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319188.422081:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319188.422096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163982753024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319188.422131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319188.422135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319188.422137:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319188.422140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4432 00000100:00080000:2.0:1767319188.422142:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319188.422146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319188.422157:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163982753280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319188.422166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319188.422169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319188.422170:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319188.422174:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4432 00000100:00080000:2.0:1767319188.422176:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319188.422179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319188.422191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdc00 x1853163982753408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319188.422200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319188.422204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319188.422205:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319188.422208:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4432 00000100:00080000:2.0:1767319188.422211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319188.422214:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319188.422224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163982753536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319188.422250:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319188.422261:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982753152:0@lo:400:kworker.0 00000100:00100000:2.0:1767319188.422489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319188.422754:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982752896:0@lo:400:kworker.0 00000100:00100000:1.0:1767319188.422785:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319188.422961:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753152 00000100:00100000:2.0:1767319188.422969:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753152 00000100:00100000:2.0:1767319188.423026:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982752896 00000100:00100000:2.0:1767319188.423030:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982752896 00000100:00100000:2.0:1767319188.423053:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965 00000100:00100000:2.0:1767319188.423062:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982753152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319188.423163:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982753152:12345-0@lo:400:kworker.0 Request processed in 102us (674us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319188.423172:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965 00000100:00100000:2.0:1767319188.423190:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 966 00000100:00100000:2.0:1767319188.423196:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982752896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319188.423229:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982752896:12345-0@lo:400:kworker.0 Request processed in 33us (443us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319188.423235:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 966 00010000:00080000:2.0:1767319188.423302:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fce00 x1853163982753152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319204 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319188.423315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982753152:0@lo:400:kworker.0 00000100:00080000:1.0:1767319188.423826:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319188.423853:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc380 x1853163982752896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319204 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319188.423863:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982752896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319188.424405:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982752768:0@lo:250:kworker.0 00000400:00000100:0.0:1767319188.424444:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982752768 offset 0 length 520: 4 00000100:00100000:0.0:1767319188.424473:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982753024:0@lo:38:kworker.0 00000100:00100000:0.0:1767319188.424499:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319188.424541:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982753280:0@lo:38:kworker.0 00000100:00100000:0.0:1767319188.424573:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319188.424595:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982753408:0@lo:38:kworker.0 00000100:00100000:0.0:1767319188.424619:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319188.424644:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982753536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319188.424663:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319188.424697:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753024 00000100:00100000:0.0:1767319188.424703:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753024 00000100:00100000:0.0:1767319188.424738:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753280 00000100:00100000:0.0:1767319188.424741:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753280 00000100:00100000:0.0:1767319188.424755:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753536 00000100:00100000:0.0:1767319188.424758:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753536 00000100:00100000:0.0:1767319188.424769:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1267 00000100:00100000:0.0:1767319188.424774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319188.424792:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319188.424830:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753024:12345-0@lo:38:kworker.0 Request processed in 54us (330us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319188.424836:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1267 00000100:00100000:0.0:1767319188.424850:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268 00000100:00100000:0.0:1767319188.424854:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319188.424863:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319188.424886:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753280:12345-0@lo:38:kworker.0 Request processed in 30us (312us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319188.424892:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268 00000100:00100000:0.0:1767319188.424903:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269 00000100:00100000:0.0:1767319188.424906:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753536:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319188.424915:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319188.424939:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982753536:12345-0@lo:38:kworker.0 Request processed in 31us (274us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319188.424948:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269 00000100:00080000:0.0:1767319188.424976:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319188.424980:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319188.424984:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982753024:0@lo:38:kworker.0 00000100:00080000:0.0:1767319188.424992:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319188.424995:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319188.424998:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982753280:0@lo:38:kworker.0 00000100:00080000:0.0:1767319188.425006:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319188.425009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319188.425011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982753536:0@lo:38:kworker.0 00000100:00100000:3.0:1767319188.425311:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753408 00000100:00100000:3.0:1767319188.425316:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753408 00000100:00100000:3.0:1767319188.425355:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 342 00000100:00100000:3.0:1767319188.425361:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982753408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319188.425374:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319188.425419:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982753408:12345-0@lo:38:kworker.0 Request processed in 57us (798us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319188.425425:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 342 00000100:00080000:1.0:1767319188.425570:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319188.425575:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319188.425578:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982753408:0@lo:38:kworker.0 00010000:00010000:0.0:1767319188.612471:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319189.636522:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319190.660761:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319191.439909:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319191.439947:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319191.440008:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611136 00000100:00100000:0.0:1767319191.440019:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611136 00000100:00100000:0.0:1767319191.440079:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611008 00000100:00100000:0.0:1767319191.440082:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611008 00000100:00100000:0.0:1767319191.440101:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 967 00000100:00100000:0.0:1767319191.440111:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915611136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319191.440177:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915611136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 67us (276us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319191.440185:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 967 00000100:00100000:0.0:1767319191.440199:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1270 00000100:00100000:0.0:1767319191.440203:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915611008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319191.440220:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319191.440239:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915611008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 35us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319191.440246:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1270 00000100:00080000:1.0:1767319191.440772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319191.684417:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319192.708561:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319193.028714:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319193.028738:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982753664:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319193.028814:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319193.028862:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982753792:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319193.028895:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319193.028972:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753664 00000100:00100000:1.0:1767319193.028981:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753664 00000100:00100000:1.0:1767319193.029041:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753792 00000100:00100000:1.0:1767319193.029045:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753792 00000100:00100000:1.0:1767319193.029064:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 662 00000100:00100000:1.0:1767319193.029073:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982753664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319193.029102:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319193.029152:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982753664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (338us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319193.029160:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 662 00000100:00100000:1.0:1767319193.029177:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 663 00000100:00100000:1.0:1767319193.029183:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982753792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319193.029197:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319193.029234:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982753792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319193.029251:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 663 00000100:00100000:1.0:1767319193.029325:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982753664:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319193.029395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319193.029400:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319193.029424:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982753792:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319193.541332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319193.541342:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319193.541405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319193.541409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319193.541412:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319193.541421:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4436 00000100:00080000:3.0:1767319193.541433:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319193.541449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319193.541471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3b80 x1853163982754048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319193.541515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319193.541519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319193.541520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319193.541524:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4436 00000100:00080000:3.0:1767319193.541527:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319193.541531:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319193.541542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1f80 x1853163982754304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319193.541552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319193.541555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319193.541556:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319193.541560:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4436 00000100:00080000:3.0:1767319193.541562:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319193.541565:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319193.541575:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1c00 x1853163982754432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319193.541595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319193.541598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319193.541610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319193.541619:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4436 00000100:00080000:3.0:1767319193.541622:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319193.541625:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319193.541635:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982754816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319193.542132:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982754560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319193.542202:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542264:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319193.542302:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982754176:0@lo:400:kworker.0 00000100:00100000:2.0:1767319193.542328:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542360:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754176 00000100:00100000:2.0:1767319193.542367:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754176 00000100:00100000:2.0:1767319193.542419:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968 00000100:00100000:2.0:1767319193.542428:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982754176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319193.542476:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982754176:12345-0@lo:400:kworker.0 Request processed in 50us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319193.542483:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968 00000100:00080000:2.0:1767319193.542502:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319193.542528:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0380 x1853163982754176/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319209 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319193.542540:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982754176:0@lo:400:kworker.0 00000100:00100000:2.0:1767319193.542569:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982754048:0@lo:38:kworker.0 00000100:00100000:2.0:1767319193.542592:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542607:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982754304:0@lo:38:kworker.0 00000100:00100000:2.0:1767319193.542625:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542638:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982754432:0@lo:38:kworker.0 00000100:00100000:2.0:1767319193.542656:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542669:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982754816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319193.542686:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319193.542707:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754048 00000100:00100000:2.0:1767319193.542711:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754048 00000100:00100000:2.0:1767319193.542724:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754304 00000100:00100000:2.0:1767319193.542726:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754304 00000100:00100000:2.0:1767319193.542732:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754816 00000100:00100000:2.0:1767319193.542734:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754816 00000100:00100000:2.0:1767319193.542742:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271 00000100:00100000:2.0:1767319193.542746:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319193.542762:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319193.542789:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754048:12345-0@lo:38:kworker.0 Request processed in 42us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319193.542796:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271 00000100:00100000:2.0:1767319193.542806:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272 00000100:00100000:2.0:1767319193.542809:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319193.542817:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319193.542837:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754304:12345-0@lo:38:kworker.0 Request processed in 26us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319193.542842:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272 00000100:00100000:2.0:1767319193.542851:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1273 00000100:00100000:2.0:1767319193.542854:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319193.542863:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319193.542882:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982754816:12345-0@lo:38:kworker.0 Request processed in 27us (195us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319193.542887:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1273 00000100:00080000:2.0:1767319193.542909:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319193.542914:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319193.542917:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982754048:0@lo:38:kworker.0 00000100:00080000:2.0:1767319193.542924:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319193.542927:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319193.542930:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982754304:0@lo:38:kworker.0 00000100:00080000:2.0:1767319193.542937:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319193.542940:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319193.542943:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982754816:0@lo:38:kworker.0 00000100:00100000:3.0:1767319193.543068:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754560 00000100:00100000:3.0:1767319193.543073:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754560 00000100:00100000:3.0:1767319193.543102:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 621 00000100:00100000:3.0:1767319193.543109:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982754560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319193.543148:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982754560:12345-0@lo:400:kworker.0 Request processed in 39us (946us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319193.543154:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 621 00000100:00100000:0.0:1767319193.543468:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982754688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319193.543507:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319193.543548:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319193.543555:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982753920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319193.543577:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319193.543595:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754688 00000100:00100000:3.0:1767319193.543601:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754688 00000100:00100000:0.0:1767319193.543613:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982753920 00000100:00100000:0.0:1767319193.543618:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982753920 00000100:00100000:3.0:1767319193.543633:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 622 00000100:00100000:3.0:1767319193.543643:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982754688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319193.543654:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969 00000100:00100000:0.0:1767319193.543663:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982753920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319193.543686:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982754688:12345-0@lo:400:kworker.0 Request processed in 44us (177us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319193.543693:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 622 00000100:00100000:0.0:1767319193.543705:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982753920:12345-0@lo:400:kworker.0 Request processed in 42us (125us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319193.543712:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969 00010000:00080000:3.0:1767319193.543733:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0700 x1853163982753920/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319209 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319193.543739:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754432 00000100:00100000:0.0:1767319193.543744:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754432 00000100:00100000:3.0:1767319193.543750:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982753920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319193.543763:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982754688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319193.543768:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 343 00000100:00100000:0.0:1767319193.543774:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982754432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319193.543785:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319193.543817:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982754432:12345-0@lo:38:kworker.0 Request processed in 42us (1159us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319193.543824:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 343 00000100:00080000:2.0:1767319193.544987:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319193.545026:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982754560:0@lo:400:kworker.0 00000100:00080000:2.0:1767319193.545064:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319193.545069:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319193.545072:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982754432:0@lo:38:kworker.0 00010000:00010000:1.0:1767319193.732491:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319194.756417:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319195.780501:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319196.561339:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319196.561389:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319196.561467:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611392 00000100:00100000:2.0:1767319196.561478:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611392 00000100:00100000:2.0:1767319196.561563:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611264 00000100:00100000:2.0:1767319196.561567:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611264 00000100:00100000:2.0:1767319196.561586:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 970 00000100:00100000:2.0:1767319196.561596:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915611392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319196.561673:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915611392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319196.561681:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 970 00000100:00100000:2.0:1767319196.561699:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1274 00000100:00100000:2.0:1767319196.561703:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915611264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319196.561723:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319196.561748:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915611264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (359us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319196.561753:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1274 00000100:00080000:0.0:1767319196.561925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319196.804667:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319197.829345:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319198.148780:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982754944:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319198.148868:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319198.149100:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982754944 00000100:00100000:0.0:1767319198.149108:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982754944 00000100:00100000:0.0:1767319198.149189:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 664 00000100:00100000:0.0:1767319198.149200:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982754944:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319198.149231:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319198.149308:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982754944:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319198.149315:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 664 00000100:00100000:0.0:1767319198.149377:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982754944:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319198.149448:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982755072:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319198.149474:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319198.149869:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319198.149921:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755072 00000100:00100000:0.0:1767319198.149927:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755072 00000100:00100000:0.0:1767319198.150025:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 665 00000100:00100000:0.0:1767319198.150033:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982755072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319198.150052:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319198.150106:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982755072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (630us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319198.150113:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 665 00000100:00080000:2.0:1767319198.150203:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319198.150597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982755072:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319198.661632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319198.661644:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319198.661707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319198.661711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319198.661715:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319198.661722:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4442 00000100:00080000:3.0:1767319198.661727:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319198.661733:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319198.661754:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0700 x1853163982755328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319198.661793:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767319198.661797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319198.661802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319198.661805:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319198.661814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982755200:0@lo:400:kworker.0 00000100:00080000:3.0:1767319198.661825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4442 00000100:00080000:3.0:1767319198.661829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319198.661834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319198.661847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1500 x1853163982755584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319198.661857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319198.661860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319198.661862:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319198.661865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4442 00000100:00080000:3.0:1767319198.661867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319198.661871:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319198.661882:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3480 x1853163982755712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319198.661901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319198.661905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319198.661906:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319198.661909:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4442 00000100:00080000:3.0:1767319198.661912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319198.661915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319198.661924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163982756096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319198.661986:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319198.661997:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982755968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319198.662086:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319198.662147:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755968 00000100:00100000:2.0:1767319198.662156:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755968 00000100:00100000:2.0:1767319198.662208:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 623 00000100:00100000:2.0:1767319198.662217:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982755968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319198.662262:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982755968:12345-0@lo:400:kworker.0 Request processed in 46us (177us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319198.662288:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 623 00000100:00100000:2.0:1767319198.662332:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982755968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319198.662406:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982755328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319198.662435:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319198.662461:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982755584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319198.662465:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982755840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319198.662490:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319198.662511:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319198.662533:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982755712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319198.662537:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319198.662546:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982755456:0@lo:400:kworker.0 00000100:00100000:2.0:1767319198.662565:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319198.662581:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319198.662600:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982756096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319198.662627:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755328 00000100:00100000:2.0:1767319198.662628:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319198.662633:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755328 00000100:00100000:0.0:1767319198.662661:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755584 00000100:00100000:0.0:1767319198.662664:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755584 00000100:00100000:2.0:1767319198.662666:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755456 00000100:00100000:2.0:1767319198.662672:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755456 00000100:00100000:0.0:1767319198.662676:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756096 00000100:00100000:0.0:1767319198.662681:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756096 00000100:00100000:0.0:1767319198.662694:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275 00000100:00100000:0.0:1767319198.662700:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982755328:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319198.662711:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971 00000100:00100000:2.0:1767319198.662718:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982755456:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767319198.662722:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319198.662774:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982755328:12345-0@lo:38:kworker.0 Request processed in 73us (338us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319198.662777:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982755456:12345-0@lo:400:kworker.0 Request processed in 58us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319198.663044:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275 00000100:00100000:3.0:1767319198.663124:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755712 00000100:00100000:3.0:1767319198.663129:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755712 00000100:00100000:3.0:1767319198.663162:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 344 00000100:00100000:3.0:1767319198.663168:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982755712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319198.663179:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319198.663211:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982755712:12345-0@lo:38:kworker.0 Request processed in 43us (647us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319198.663218:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 344 00000100:00100000:1.0:1767319198.665840:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319198.665934:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755200 00000100:00100000:1.0:1767319198.665944:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755200 00000100:00100000:2.0:1767319198.666294:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971 00000100:00100000:0.0:1767319198.666324:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1276 00000100:00100000:0.0:1767319198.666334:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982755584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319198.666359:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319198.666420:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982755584:12345-0@lo:38:kworker.0 Request processed in 86us (3930us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319198.666427:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1276 00000100:00100000:0.0:1767319198.666449:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277 00000100:00100000:0.0:1767319198.666454:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982756096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319198.666463:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319198.666487:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982756096:12345-0@lo:38:kworker.0 Request processed in 32us (3860us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319198.666493:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277 00010000:00080000:0.0:1767319198.666530:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1180 x1853163982755456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319214 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319198.666544:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982755456:0@lo:400:kworker.0 00000100:00080000:0.0:1767319198.666710:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319198.666736:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982755840 00000100:00100000:0.0:1767319198.666740:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982755840 00000100:00100000:0.0:1767319198.666782:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 624 00000100:00100000:1.0:1767319198.667023:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972 00000100:00100000:1.0:1767319198.667033:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982755200:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319198.667064:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982755840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319198.667088:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982755200:12345-0@lo:400:kworker.0 Request processed in 56us (1256us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319198.667097:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972 00000100:00100000:0.0:1767319198.667115:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982755840:12345-0@lo:400:kworker.0 Request processed in 55us (4604us total) trans 0 rc 0/0 00000100:00080000:1.0:1767319198.667120:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319198.667124:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 624 00010000:00080000:1.0:1767319198.667159:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3100 x1853163982755200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319214 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319198.667171:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982755200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319198.667174:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982755840:0@lo:400:kworker.0 00000100:00080000:1.0:1767319198.667313:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319198.667319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319198.667323:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982755328:0@lo:38:kworker.0 00000100:00080000:1.0:1767319198.667333:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319198.667337:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319198.667341:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982755584:0@lo:38:kworker.0 00000100:00080000:1.0:1767319198.667352:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319198.667355:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319198.667358:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982755712:0@lo:38:kworker.0 00000100:00080000:1.0:1767319198.667367:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319198.667370:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319198.667372:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982756096:0@lo:38:kworker.0 00010000:00010000:1.0:1767319198.852866:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319199.876741:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319200.900471:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319201.676134:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319201.676232:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319201.676310:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611648 00000100:00100000:3.0:1767319201.676320:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611648 00000100:00100000:3.0:1767319201.676405:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611520 00000100:00100000:3.0:1767319201.676408:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611520 00000100:00100000:3.0:1767319201.676436:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 973 00000100:00100000:3.0:1767319201.676446:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915611648:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319201.676523:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915611648:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (400us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319201.676531:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 973 00000100:00100000:3.0:1767319201.676550:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1278 00000100:00100000:3.0:1767319201.676555:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915611520:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319201.676574:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319201.676601:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915611520:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (369us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319201.676607:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1278 00000100:00080000:1.0:1767319201.676980:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319201.926417:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319202.948551:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319203.269658:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319203.269977:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319203.269998:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982756352:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319203.270111:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319203.270185:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756352 00000100:00100000:3.0:1767319203.270194:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756352 00000100:00100000:3.0:1767319203.270264:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 666 00000100:00100000:3.0:1767319203.270296:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982756352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319203.270325:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319203.270371:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982756352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (259us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319203.270378:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 666 00000100:00080000:3.0:1767319203.270400:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319203.270432:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982756352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319203.272421:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982756224:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319203.272817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319203.273126:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756224 00000100:00100000:0.0:1767319203.273135:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756224 00000100:00100000:0.0:1767319203.273606:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 667 00000100:00100000:0.0:1767319203.273617:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982756224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319203.273647:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319203.273710:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982756224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (893us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319203.273718:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 667 00000100:00100000:2.0:1767319203.273818:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982756224:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319203.280605:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319203.783348:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319203.783360:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319203.783468:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982756480:0@lo:400:kworker.0 00000100:00100000:3.0:1767319203.783542:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319203.784814:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756480 00000100:00100000:2.0:1767319203.784827:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756480 00000100:00100000:2.0:1767319203.784913:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 974 00000100:00100000:2.0:1767319203.784925:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982756480:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319203.785028:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319203.785047:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319203.785053:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319203.785057:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319203.785062:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4447 00000100:00080000:0.0:1767319203.785066:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319203.785072:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319203.785099:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596680 x1853163982756608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319203.785146:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319203.785149:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319203.785151:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319203.785155:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4447 00000100:00080000:0.0:1767319203.785157:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319203.785160:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319203.785171:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0e00 x1853163982756864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319203.785180:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319203.785183:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319203.785185:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319203.785187:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4447 00000100:00080000:0.0:1767319203.785190:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319203.785192:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319203.785203:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2d80 x1853163982756992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319203.785268:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982757248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319203.785329:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.785372:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757248 00000100:00100000:1.0:1767319203.785376:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757248 00000100:00100000:1.0:1767319203.785597:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 625 00000100:00100000:1.0:1767319203.785605:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982757248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319203.785644:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982757248:12345-0@lo:400:kworker.0 Request processed in 40us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319203.785651:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 625 00000100:00100000:1.0:1767319203.785677:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982756608:0@lo:38:kworker.0 00000100:00100000:1.0:1767319203.785702:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.785724:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982756864:0@lo:38:kworker.0 00000100:00100000:1.0:1767319203.785744:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.785763:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982756992:0@lo:38:kworker.0 00000100:00100000:1.0:1767319203.785785:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.785818:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756992 00000100:00100000:1.0:1767319203.785823:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756992 00000100:00100000:1.0:1767319203.785842:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 345 00000100:00100000:1.0:1767319203.785847:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982756992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319203.785863:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319203.785896:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982756992:12345-0@lo:38:kworker.0 Request processed in 48us (110us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319203.785902:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 345 00000100:00100000:1.0:1767319203.785922:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756608 00000100:00100000:1.0:1767319203.785925:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756608 00000100:00100000:1.0:1767319203.785942:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756864 00000100:00100000:1.0:1767319203.785945:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756864 00000100:00100000:1.0:1767319203.785960:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1279 00000100:00100000:1.0:1767319203.785965:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982756608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319203.785974:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319203.785998:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982756608:12345-0@lo:38:kworker.0 Request processed in 32us (295us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319203.786003:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1279 00000100:00100000:1.0:1767319203.786015:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280 00000100:00100000:1.0:1767319203.786019:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982756864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319203.786028:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319203.786064:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982756864:12345-0@lo:38:kworker.0 Request processed in 42us (318us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319203.786069:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280 00000100:00100000:1.0:1767319203.786106:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982757248:0@lo:400:kworker.0 00000100:00080000:1.0:1767319203.786131:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319203.786136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319203.786139:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982756608:0@lo:38:kworker.0 00000100:00080000:1.0:1767319203.786147:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319203.786150:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319203.786153:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982756864:0@lo:38:kworker.0 00000100:00080000:1.0:1767319203.786159:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319203.786162:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319203.786165:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982756992:0@lo:38:kworker.0 00000100:00100000:1.0:1767319203.786202:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982757120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319203.786241:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.786263:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319203.786269:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982756736:0@lo:400:kworker.0 00000100:00100000:1.0:1767319203.786309:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319203.786333:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757120 00000100:00100000:1.0:1767319203.786337:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757120 00000100:00100000:1.0:1767319203.786362:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 626 00000100:00100000:1.0:1767319203.786368:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463934bc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982757120:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319203.786427:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319203.786432:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319203.786434:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319203.786439:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4447 00000100:00080000:0.0:1767319203.786442:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319203.786446:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319203.786464:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163982757376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319203.786499:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982757376:0@lo:38:kworker.0 00010000:00080000:3.0:1767319203.788444:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597100 x1853163982756480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319219 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319203.788460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982756480:0@lo:400:kworker.0 00000100:00100000:3.0:1767319203.788504:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982756736 00000100:00100000:3.0:1767319203.788511:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982756736 00000100:00100000:3.0:1767319203.788573:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975 00000100:00100000:3.0:1767319203.788582:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982756736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319203.791810:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982756480:12345-0@lo:400:kworker.0 Request processed in 6878us (8257us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319203.791812:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463934bc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982757120:12345-0@lo:400:kworker.0 Request processed in 5440us (5566us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319203.791834:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 626 00000100:00100000:2.0:1767319203.791835:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 974 00000100:00080000:2.0:1767319203.791943:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319203.791994:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982757120:0@lo:400:kworker.0 00000100:00100000:0.0:1767319203.792106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319203.792199:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757376 00000100:00100000:2.0:1767319203.792347:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757376 00000100:00100000:2.0:1767319203.792383:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281 00000100:00100000:2.0:1767319203.792390:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982757376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319203.792410:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319203.792454:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982757376:12345-0@lo:38:kworker.0 Request processed in 63us (348us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319203.792460:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281 00000100:00080000:0.0:1767319203.792523:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319203.792528:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319203.792532:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982757376:0@lo:38:kworker.0 00000100:00100000:3.0:1767319203.792654:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982756736:12345-0@lo:400:kworker.0 Request processed in 4071us (6343us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319203.792662:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975 00010000:00080000:2.0:1767319203.792719:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0380 x1853163982756736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319219 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319203.792731:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982756736:0@lo:400:kworker.0 00010000:00010000:0.0:1767319203.972423:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319204.996565:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319206.025619:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319206.796116:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319206.796226:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319206.796315:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611904 00000100:00100000:1.0:1767319206.796326:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611904 00000100:00100000:1.0:1767319206.796382:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915611776 00000100:00100000:1.0:1767319206.796386:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915611776 00000100:00100000:1.0:1767319206.796414:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 976 00000100:00100000:1.0:1767319206.796424:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915611904:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319206.796504:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915611904:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319206.796512:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 976 00000100:00100000:1.0:1767319206.796533:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1282 00000100:00100000:1.0:1767319206.796538:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915611776:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319206.796556:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319206.796581:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915611776:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (356us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319206.796587:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1282 00000100:00080000:2.0:1767319206.798545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319207.045130:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319208.068507:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319208.388568:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319208.388591:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982757504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319208.388653:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319208.388911:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757504 00000100:00100000:3.0:1767319208.388920:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757504 00000100:00100000:3.0:1767319208.389077:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 668 00000100:00100000:3.0:1767319208.389086:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982757504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319208.389114:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319208.389159:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982757504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319208.389165:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 668 00000100:00100000:1.0:1767319208.389293:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982757504:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319208.394984:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319208.395140:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319208.395160:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982757632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319208.395229:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319208.395459:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757632 00000100:00100000:1.0:1767319208.395468:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757632 00000100:00100000:1.0:1767319208.395527:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 669 00000100:00100000:1.0:1767319208.395536:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982757632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319208.395565:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319208.395612:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982757632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (383us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319208.395619:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 669 00000100:00100000:1.0:1767319208.395681:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982757632:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319208.397407:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319208.900420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319208.900433:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319208.900502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319208.900507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319208.900511:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319208.900520:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4452 00000100:00080000:0.0:1767319208.900525:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319208.900532:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319208.900567:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0e00 x1853163982757888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319208.900605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319208.900609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319208.900611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319208.900615:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4452 00000100:00080000:0.0:1767319208.900617:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319208.900621:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319208.900635:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0380 x1853163982758144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319208.900644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319208.900647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319208.900649:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319208.900653:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4452 00000100:00080000:0.0:1767319208.900655:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319208.900659:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319208.900664:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982757888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.900676:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988700 x1853163982758272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319208.900704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319208.900709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319208.900710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319208.900714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4452 00000100:00080000:0.0:1767319208.900717:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319208.900720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319208.900731:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982758656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319208.900750:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319208.900759:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982758400:0@lo:400:kworker.0 00000100:00100000:0.0:1767319208.900800:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319208.900845:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757888 00000100:00100000:0.0:1767319208.900852:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757888 00000100:00100000:0.0:1767319208.900900:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283 00000100:00100000:2.0:1767319208.900902:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982758144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.900910:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982757888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319208.900931:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319208.900933:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319208.900992:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982757888:12345-0@lo:38:kworker.0 Request processed in 82us (241us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319208.900999:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982758272:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.901003:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283 00000100:00100000:0.0:1767319208.901025:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758144 00000100:00100000:0.0:1767319208.901030:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758144 00000100:00100000:2.0:1767319208.901031:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319208.901059:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284 00000100:00100000:2.0:1767319208.901063:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982758656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.901068:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982758144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319208.901080:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319208.901086:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319208.901110:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982758144:12345-0@lo:38:kworker.0 Request processed in 43us (179us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319208.901116:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284 00000100:00100000:0.0:1767319208.901148:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319208.901155:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982758016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319208.901185:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319208.901202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319208.901208:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319208.901213:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982757888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.901217:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758656 00000100:00100000:0.0:1767319208.901222:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758656 00000100:00080000:2.0:1767319208.901224:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319208.901229:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319208.901232:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982758144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.901240:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758016 00000100:00100000:0.0:1767319208.901245:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758016 00000100:00100000:2.0:1767319208.901289:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1285 00000100:00100000:2.0:1767319208.901296:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982758656:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319208.901305:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977 00010000:00020000:2.0:1767319208.901309:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319208.901317:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982758016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319208.901341:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982758656:12345-0@lo:38:kworker.0 Request processed in 43us (254us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319208.901348:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1285 00000100:00100000:0.0:1767319208.901368:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982758016:12345-0@lo:400:kworker.0 Request processed in 54us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319208.901376:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977 00000100:00080000:0.0:1767319208.901395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319208.901414:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3800 x1853163982758016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319224 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319208.901425:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982758016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319208.901494:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982758528:0@lo:400:kworker.0 00000100:00100000:3.0:1767319208.901525:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319208.901871:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319208.901878:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982757760:0@lo:400:kworker.0 00000100:00100000:3.0:1767319208.902188:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319208.902230:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319208.902234:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319208.902238:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982758656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319208.902300:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758400 00000100:00100000:1.0:1767319208.902305:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758400 00000100:00100000:1.0:1767319208.902339:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758528 00000100:00100000:1.0:1767319208.902342:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758528 00000100:00100000:1.0:1767319208.902362:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 627 00000100:00100000:1.0:1767319208.902368:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982758400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319208.902411:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982758400:12345-0@lo:400:kworker.0 Request processed in 42us (1610us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319208.902417:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 627 00000100:00100000:1.0:1767319208.902430:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 628 00000100:00100000:1.0:1767319208.902435:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982758528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319208.902462:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982758528:12345-0@lo:400:kworker.0 Request processed in 26us (936us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319208.902468:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 628 00000100:00100000:1.0:1767319208.902485:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758272 00000100:00100000:1.0:1767319208.902488:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758272 00000100:00100000:1.0:1767319208.902503:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 346 00000100:00100000:1.0:1767319208.902507:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982758272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319208.902522:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319208.902548:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982758272:12345-0@lo:38:kworker.0 Request processed in 40us (1519us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319208.902553:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 346 00000100:00100000:2.0:1767319208.902617:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982757760 00000100:00100000:2.0:1767319208.902622:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982757760 00000100:00100000:2.0:1767319208.902662:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 978 00000100:00100000:2.0:1767319208.902669:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982757760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319208.902706:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982757760:12345-0@lo:400:kworker.0 Request processed in 37us (517us total) trans 0 rc 0/0 00010000:00080000:0.0:1767319208.902723:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a2d80 x1853163982757760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319224 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319208.902734:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982757760:0@lo:400:kworker.0 00000100:00100000:0.0:1767319208.902743:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982758528:0@lo:400:kworker.0 00000100:00100000:0.0:1767319208.902768:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982758400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319208.902778:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 978 00000100:00080000:0.0:1767319208.902797:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319208.903010:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319208.903015:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319208.903018:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982758272:0@lo:38:kworker.0 00010000:00010000:1.0:1767319209.093048:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319210.116415:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319210.892787:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915612032 offset 0 length 520: 4 00000100:00100000:3.0:1767319210.892953:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319210.893037:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319210.893093:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612160 00000100:00100000:0.0:1767319210.893113:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612160 00000100:00100000:0.0:1767319210.893160:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612288 00000100:00100000:0.0:1767319210.893164:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612288 00000100:00100000:0.0:1767319210.893219:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 979 00000100:00100000:0.0:1767319210.893230:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915612288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319210.893359:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915612288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319210.893367:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 979 00000100:00100000:0.0:1767319210.893386:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1286 00000100:00100000:0.0:1767319210.893391:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915612160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319210.893411:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319210.893416:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319210.893439:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915612160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (488us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319210.893447:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1286 00010000:00010000:1.0:1767319211.143010:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319212.164511:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319213.188513:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319213.508744:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319213.508768:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982758912:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319213.508930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319213.509080:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758912 00000100:00100000:3.0:1767319213.509090:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758912 00000100:00100000:3.0:1767319213.509149:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 670 00000100:00100000:3.0:1767319213.509158:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982758912:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319213.509189:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319213.509235:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982758912:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (389us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319213.509245:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 670 00000100:00080000:3.0:1767319213.509267:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319213.509362:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982758912:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319213.509715:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982758784:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319213.509750:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319213.510972:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982758784 00000100:00100000:1.0:1767319213.510982:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982758784 00000100:00080000:3.0:1767319213.511133:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319213.511193:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00100000:1.0:1767319213.511203:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982758784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319213.511229:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319213.511318:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982758784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (1567us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319213.511325:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00100000:2.0:1767319213.511819:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982758784:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319214.020386:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319188/real 1767319188] req@ffff9346100ff800 x1853163982752768/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319213 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319214.020420:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100ff800 x1853163982752768/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319213 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319214.020441:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319214.020463:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319214.020467:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982752768:0@lo:250:kworker.0 00000100:00080000:0.0:1767319214.020703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319214.020709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319214.020712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319214.020717:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4436 00000100:00080000:0.0:1767319214.020721:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319214.020727:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319214.020759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989500 x1853163982759040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319214.020913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319214.020918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319214.020920:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319214.020924:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4457 00000100:00080000:0.0:1767319214.020926:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319214.020931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319214.020947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982759296/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319214.020972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319214.020976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319214.020977:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319214.020982:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4457 00000100:00080000:0.0:1767319214.020985:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319214.020989:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319214.021003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163982759552/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319214.021012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319214.021015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319214.021016:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319214.021020:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4457 00000100:00080000:0.0:1767319214.021022:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319214.021025:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319214.021036:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964e00 x1853163982759680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319214.021045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319214.021049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319214.021050:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319214.021053:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4457 00000100:00080000:0.0:1767319214.021056:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319214.021060:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319214.021069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982759808/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319214.021145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982759168:0@lo:400:kworker.0 00000100:00100000:1.0:1767319214.021221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319214.021448:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759168 00000100:00100000:0.0:1767319214.021457:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759168 00000100:00100000:2.0:1767319214.021635:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982759424:0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.021666:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319214.021724:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759424 00000100:00100000:2.0:1767319214.021728:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759424 00000100:00100000:3.0:1767319214.021843:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982759040:0@lo:250:kworker.0 00000400:00000100:3.0:1767319214.021875:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982759040 offset 0 length 520: 4 00000100:00100000:3.0:1767319214.021896:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982759296:0@lo:38:kworker.0 00000100:00100000:3.0:1767319214.021919:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319214.021938:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982759552:0@lo:38:kworker.0 00000100:00100000:3.0:1767319214.021958:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319214.021974:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982759680:0@lo:38:kworker.0 00000100:00100000:3.0:1767319214.021999:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319214.022016:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982759808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319214.022035:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319214.022064:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759296 00000100:00100000:3.0:1767319214.022068:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759296 00000100:00100000:3.0:1767319214.022092:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759552 00000100:00100000:3.0:1767319214.022095:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759552 00000100:00100000:3.0:1767319214.022106:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287 00000100:00100000:3.0:1767319214.022112:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759296:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319214.022133:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319214.022175:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759296:12345-0@lo:38:kworker.0 Request processed in 62us (255us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319214.022182:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287 00000100:00100000:3.0:1767319214.022199:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759808 00000100:00100000:3.0:1767319214.022202:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759808 00000100:00100000:3.0:1767319214.022213:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1288 00000100:00100000:3.0:1767319214.022216:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319214.022225:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319214.022246:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759552:12345-0@lo:38:kworker.0 Request processed in 28us (287us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319214.022252:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1288 00000100:00100000:3.0:1767319214.022262:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289 00000100:00100000:3.0:1767319214.022266:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319214.022292:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319214.022326:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982759808:12345-0@lo:38:kworker.0 Request processed in 58us (290us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319214.022332:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289 00000100:00080000:3.0:1767319214.022367:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319214.022372:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319214.022375:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982759296:0@lo:38:kworker.0 00000100:00080000:3.0:1767319214.022385:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319214.022388:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319214.022391:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982759552:0@lo:38:kworker.0 00000100:00080000:3.0:1767319214.022397:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319214.022399:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319214.022402:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982759808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319214.022443:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759680 00000100:00100000:3.0:1767319214.022447:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759680 00000100:00100000:3.0:1767319214.022467:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 347 00000100:00100000:3.0:1767319214.022471:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982759680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319214.022481:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319214.022510:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982759680:12345-0@lo:38:kworker.0 Request processed in 38us (510us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319214.022515:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 347 00000100:00080000:3.0:1767319214.022535:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319214.022538:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319214.022541:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982759680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319214.023027:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980 00000100:00100000:0.0:1767319214.023035:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982759168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319214.023087:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982759168:12345-0@lo:400:kworker.0 Request processed in 54us (1866us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319214.023093:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980 00010000:00080000:0.0:1767319214.023142:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988700 x1853163982759168/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319230 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319214.023155:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982759168:0@lo:400:kworker.0 00000100:00080000:1.0:1767319214.023517:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319214.023752:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981 00000100:00100000:2.0:1767319214.023763:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982759424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319214.023913:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982759424:12345-0@lo:400:kworker.0 Request processed in 153us (2246us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319214.023920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981 00010000:00080000:1.0:1767319214.023980:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964380 x1853163982759424/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319230 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319214.023990:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982759424:0@lo:400:kworker.0 00010000:00010000:1.0:1767319214.215564:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319215.238089:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319216.012647:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319216.012698:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319216.012952:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612544 00000100:00100000:2.0:1767319216.012971:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612544 00000100:00100000:2.0:1767319216.013050:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612416 00000100:00100000:2.0:1767319216.013053:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612416 00000100:00100000:2.0:1767319216.013074:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 982 00000100:00100000:2.0:1767319216.013082:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915612544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319216.013152:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915612544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 70us (516us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319216.013168:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 982 00000100:00100000:2.0:1767319216.013185:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1290 00000100:00100000:2.0:1767319216.013191:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915612416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319216.013208:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319216.013576:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915612416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 383us (876us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319216.013584:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1290 00000100:00080000:2.0:1767319216.013614:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319216.260508:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319217.284558:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319218.308907:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319218.628911:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319218.628928:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982759936:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319218.628999:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319218.629292:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982759936 00000100:00100000:2.0:1767319218.629300:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982759936 00000100:00100000:2.0:1767319218.629363:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00100000:2.0:1767319218.629372:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982759936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319218.629409:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319218.629454:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982759936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319218.629603:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982759936:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319218.629744:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00080000:0.0:1767319218.629971:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319218.630368:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319218.630379:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982760064:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319218.630426:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319218.630741:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760064 00000100:00100000:0.0:1767319218.630746:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760064 00000100:00100000:0.0:1767319218.630803:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 673 00000100:00100000:0.0:1767319218.630810:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982760064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319218.630826:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319218.630859:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982760064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (433us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319218.630866:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 673 00000100:00080000:1.0:1767319218.630924:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319218.631102:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982760064:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319219.140621:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.140632:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319219.140689:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.140693:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.140696:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.140704:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.140709:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.140715:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319219.140741:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5c00 x1853163982760320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319219.140794:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.140797:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.140799:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.140803:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.140806:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.140810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319219.140822:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5f80 x1853163982760576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319219.140831:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.140834:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.140835:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.140838:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4462 00000100:00100000:0.0:1767319219.140840:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982760320:0@lo:38:kworker.0 00000100:00080000:3.0:1767319219.140840:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.140843:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319219.140853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7800 x1853163982760704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319219.140866:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319219.140869:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319219.140870:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319219.140873:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4462 00000100:00080000:3.0:1767319219.140874:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319219.140877:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319219.140885:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982760832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319219.140907:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319219.140915:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982760448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319219.140919:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319219.140977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982760192:0@lo:400:kworker.0 00000100:00100000:2.0:1767319219.141124:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760320 00000100:00100000:2.0:1767319219.141131:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760320 00000100:00100000:2.0:1767319219.141165:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1291 00000100:00100000:2.0:1767319219.141172:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982760320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319219.141190:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319219.154900:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319219.154980:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760448 00000100:00100000:3.0:1767319219.154990:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760448 00000100:00100000:1.0:1767319219.155041:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319219.155050:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760192 00000100:00100000:3.0:1767319219.155053:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760192 00000100:00100000:3.0:1767319219.155072:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983 00000100:00080000:1.0:1767319219.155076:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319219.155080:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982760448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319219.155141:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982760448:12345-0@lo:400:kworker.0 Request processed in 62us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319219.155148:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983 00000100:00100000:3.0:1767319219.155164:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984 00000100:00100000:3.0:1767319219.155170:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982760192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319219.155202:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982760192:12345-0@lo:400:kworker.0 Request processed in 32us (161us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319219.155207:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984 00010000:00080000:1.0:1767319219.155235:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988000 x1853163982760192/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319235 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:3.0:1767319219.155235:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7480 x1853163982760448/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319235 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319219.155250:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982760192:0@lo:400:kworker.0 00000100:00100000:3.0:1767319219.155250:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982760448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319219.162155:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982760576:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.162230:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.162294:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982760704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.162322:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.162426:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982760832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319219.162451:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319219.162494:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760576 00000100:00100000:0.0:1767319219.162513:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760576 00000100:00100000:0.0:1767319219.162569:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760832 00000100:00100000:0.0:1767319219.162572:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760832 00000100:00100000:0.0:1767319219.162591:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292 00000100:00100000:0.0:1767319219.162598:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982760576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319219.162617:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319219.162655:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982760576:12345-0@lo:38:kworker.0 Request processed in 56us (424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319219.162662:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292 00000100:00100000:0.0:1767319219.162677:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293 00000100:00100000:0.0:1767319219.162681:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982760832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319219.162690:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319219.162732:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982760832:12345-0@lo:38:kworker.0 Request processed in 49us (279us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319219.162738:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293 00000100:00080000:0.0:1767319219.162788:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319219.162793:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319219.162797:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982760832:0@lo:38:kworker.0 00000100:00080000:0.0:1767319219.162806:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319219.162809:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319219.162811:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982760576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319219.164326:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982760320:12345-0@lo:38:kworker.0 Request processed in 23147us (23399us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319219.164338:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1291 00000100:00080000:0.0:1767319219.165739:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319219.165747:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319219.165752:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982760320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319219.168417:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760704 00000100:00100000:1.0:1767319219.168433:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760704 00000100:00100000:1.0:1767319219.168720:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 348 00000100:00100000:1.0:1767319219.168729:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982760704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319219.168752:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319219.168855:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982760704:12345-0@lo:38:kworker.0 Request processed in 125us (6530us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319219.168863:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 348 00000100:00080000:0.0:1767319219.168962:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319219.168968:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319219.168972:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982760704:0@lo:38:kworker.0 00010000:00010000:0.0:1767319219.332839:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319220.356492:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319221.135563:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319221.135615:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319221.135705:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612800 00000100:00100000:2.0:1767319221.135715:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612800 00000100:00100000:2.0:1767319221.135805:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612672 00000100:00100000:2.0:1767319221.135809:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612672 00000100:00100000:2.0:1767319221.135832:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 985 00000100:00100000:2.0:1767319221.135842:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915612800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319221.135914:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915612800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (362us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319221.135923:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 985 00000100:00100000:2.0:1767319221.135940:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1294 00000100:00100000:2.0:1767319221.135945:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915612672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319221.135964:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319221.135993:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915612672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (377us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319221.136000:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1294 00000100:00080000:1.0:1767319221.136485:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319221.380496:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319222.404441:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319223.428573:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319223.748612:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319223.748933:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319223.748959:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982761088:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319223.749029:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319223.749236:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761088 00000100:00100000:1.0:1767319223.749245:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761088 00000100:00100000:1.0:1767319223.749335:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 674 00000100:00100000:1.0:1767319223.749345:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982761088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319223.749375:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319223.749433:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982761088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (404us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319223.749441:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 674 00000100:00100000:1.0:1767319223.749486:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982761088:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319223.749696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319223.749810:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982760960:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319223.749843:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319223.749934:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982760960 00000100:00100000:3.0:1767319223.749939:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982760960 00000100:00100000:3.0:1767319223.750014:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 675 00000100:00100000:3.0:1767319223.750022:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982760960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319223.750039:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319223.750081:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982760960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (237us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319223.750088:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 675 00000100:00100000:2.0:1767319223.750165:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982760960:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319223.750326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319224.260760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319224.260769:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319224.260821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319224.260824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319224.260828:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319224.260835:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4467 00000100:00080000:0.0:1767319224.260840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319224.260847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319224.260849:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319224.260867:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982761216:0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.260874:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cce00 x1853163982761344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319224.260901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319224.260904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319224.260905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319224.260909:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4467 00000100:00080000:0.0:1767319224.260911:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319224.260914:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319224.260923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cdc00 x1853163982761600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319224.260928:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319224.260932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319224.260935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319224.260937:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319224.260939:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4467 00000100:00080000:0.0:1767319224.260941:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319224.260944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319224.260953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cd180 x1853163982761728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319224.260961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319224.260964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319224.260966:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319224.260968:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4467 00000100:00100000:3.0:1767319224.260976:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761216 00000100:00080000:0.0:1767319224.260981:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319224.260983:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761216 00000100:00080000:0.0:1767319224.260985:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319224.260995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3b80 x1853163982761856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319224.261026:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319224.261037:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986 00000100:00100000:0.0:1767319224.261040:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982761472:0@lo:400:kworker.0 00000100:00100000:3.0:1767319224.261047:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982761216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.261067:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319224.261093:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982761216:12345-0@lo:400:kworker.0 Request processed in 48us (164us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319224.261099:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986 00000100:00100000:0.0:1767319224.261108:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761472 00000100:00100000:0.0:1767319224.261112:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761472 00000100:00100000:0.0:1767319224.261138:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987 00000100:00100000:0.0:1767319224.261144:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982761472:12345-0@lo:400:kworker.0 00010000:00080000:3.0:1767319224.261146:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965f80 x1853163982761216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319240 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319224.261158:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982761216:0@lo:400:kworker.0 00000100:00100000:0.0:1767319224.261175:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982761472:12345-0@lo:400:kworker.0 Request processed in 31us (106us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319224.261181:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987 00000100:00080000:3.0:1767319224.261186:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319224.261701:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cdf80 x1853163982761472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319240 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319224.261710:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982761472:0@lo:400:kworker.0 00000100:00100000:2.0:1767319224.262863:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982761344:0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.262916:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319224.262957:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982761600:0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.262992:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319224.263024:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982761728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.263049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319224.263099:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982761856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319224.263113:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761728 00000100:00100000:1.0:1767319224.263120:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761728 00000100:00100000:2.0:1767319224.263131:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319224.263149:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 349 00000100:00100000:1.0:1767319224.263156:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982761728:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.263162:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761344 00000100:00100000:2.0:1767319224.263165:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761344 00010000:00020000:1.0:1767319224.263176:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319224.263181:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761600 00000100:00100000:2.0:1767319224.263183:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761600 00000100:00100000:2.0:1767319224.263190:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761856 00000100:00100000:2.0:1767319224.263219:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761856 00000100:00100000:2.0:1767319224.263230:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295 00000100:00100000:1.0:1767319224.263231:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982761728:12345-0@lo:38:kworker.0 Request processed in 74us (180us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319224.263235:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982761344:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319224.263239:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 349 00010000:00020000:2.0:1767319224.263243:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319224.263267:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982761344:12345-0@lo:38:kworker.0 Request processed in 31us (351us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319224.263301:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295 00000100:00100000:1.0:1767319224.263308:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296 00000100:00100000:2.0:1767319224.263312:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1297 00000100:00100000:1.0:1767319224.263313:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982761600:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319224.263316:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982761856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319224.263322:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767319224.263324:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319224.263345:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982761600:12345-0@lo:38:kworker.0 Request processed in 31us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319224.263345:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982761856:12345-0@lo:38:kworker.0 Request processed in 27us (213us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319224.263351:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1297 00000100:00080000:2.0:1767319224.263392:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319224.263406:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319224.263410:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982761344:0@lo:38:kworker.0 00000100:00080000:2.0:1767319224.263421:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319224.263424:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319224.263426:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982761600:0@lo:38:kworker.0 00000100:00080000:2.0:1767319224.263434:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319224.263622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319224.263626:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982761728:0@lo:38:kworker.0 00000100:00080000:2.0:1767319224.263638:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319224.263642:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319224.263645:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982761856:0@lo:38:kworker.0 00000100:00100000:1.0:1767319224.263830:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296 00010000:00010000:1.0:1767319224.454908:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319225.476426:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319226.252324:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319226.252415:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319226.252486:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613056 00000100:00100000:2.0:1767319226.252497:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613056 00000100:00100000:2.0:1767319226.252576:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915612928 00000100:00100000:2.0:1767319226.252580:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915612928 00000100:00100000:2.0:1767319226.252602:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 988 00000100:00100000:2.0:1767319226.252611:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915613056:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319226.252692:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915613056:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (423us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319226.252701:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 988 00000100:00100000:2.0:1767319226.252719:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1298 00000100:00100000:2.0:1767319226.252724:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915612928:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319226.252743:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319226.252769:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915612928:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319226.252775:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1298 00000100:00080000:0.0:1767319226.252898:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319226.500598:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319227.528772:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319228.548429:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319228.869551:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319228.869573:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982761984:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319228.869667:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319228.869721:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982762112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319228.869753:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319228.873176:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982761984 00000100:00100000:1.0:1767319228.873190:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982761984 00000100:00100000:1.0:1767319228.873301:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762112 00000100:00100000:1.0:1767319228.873305:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762112 00000100:00100000:1.0:1767319228.873327:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 676 00000100:00100000:1.0:1767319228.873336:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982761984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319228.873366:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319228.873439:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982761984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (3772us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319228.873446:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 676 00000100:00100000:1.0:1767319228.873463:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 677 00000100:00100000:1.0:1767319228.873469:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982762112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319228.873483:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319228.873512:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982762112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (3760us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319228.873518:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 677 00000100:00100000:1.0:1767319228.873567:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982761984:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319228.873786:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319228.873791:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319228.873816:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982762112:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319229.380800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319229.380809:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319229.380871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319229.380881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319229.380885:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319229.380892:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4472 00000100:00080000:3.0:1767319229.380897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319229.380904:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319229.380926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982762368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319229.380982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319229.380986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319229.380988:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319229.380992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4472 00000100:00080000:3.0:1767319229.380994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319229.380998:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319229.381011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982762624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319229.381037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319229.381040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319229.381042:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319229.381045:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4472 00000100:00080000:3.0:1767319229.381048:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319229.381051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319229.381065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982762752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319229.381074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319229.381077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319229.381079:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319229.381081:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4472 00000100:00080000:3.0:1767319229.381083:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319229.381087:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319229.381095:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982762880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319229.381148:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982762240:0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.381208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319229.381291:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319229.381300:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982762496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319229.381307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982762368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319229.381329:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319229.381349:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319229.381427:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982762624:0@lo:38:kworker.0 00000100:00100000:1.0:1767319229.381452:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319229.388218:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982762752:0@lo:38:kworker.0 00000100:00100000:1.0:1767319229.388300:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319229.388325:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982762880:0@lo:38:kworker.0 00000100:00100000:1.0:1767319229.388363:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319229.391806:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762240 00000100:00100000:2.0:1767319229.391825:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762240 00000100:00100000:2.0:1767319229.391913:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762496 00000100:00100000:3.0:1767319229.391921:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762368 00000100:00100000:3.0:1767319229.391927:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762368 00000100:00100000:3.0:1767319229.391950:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762624 00000100:00100000:3.0:1767319229.391953:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762624 00000100:00100000:3.0:1767319229.391983:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989 00000100:00100000:3.0:1767319229.391997:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982762240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.392156:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762496 00000100:00100000:2.0:1767319229.392182:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990 00000100:00100000:2.0:1767319229.392189:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982762496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.392665:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982762496:12345-0@lo:400:kworker.0 Request processed in 476us (11336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319229.392674:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990 00000100:00100000:2.0:1767319229.392694:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1299 00000100:00100000:2.0:1767319229.392698:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982762368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.392719:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.392751:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982762368:12345-0@lo:38:kworker.0 Request processed in 52us (11404us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.392757:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1299 00000100:00100000:2.0:1767319229.392769:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1300 00000100:00100000:2.0:1767319229.392773:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982762624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.392782:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.392819:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982762624:12345-0@lo:38:kworker.0 Request processed in 45us (11367us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.392825:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1300 00010000:00080000:2.0:1767319229.392881:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965f80 x1853163982762496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319245 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319229.392895:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982762496:0@lo:400:kworker.0 00000100:00100000:2.0:1767319229.393412:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762752 00000100:00100000:2.0:1767319229.393419:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762752 00000100:00100000:2.0:1767319229.393505:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 350 00000100:00100000:2.0:1767319229.393510:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982762752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319229.393520:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319229.393555:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982762752:12345-0@lo:38:kworker.0 Request processed in 44us (5256us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319229.393562:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 350 00000100:00080000:2.0:1767319229.393598:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319229.393605:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982762880 00000100:00100000:0.0:1767319229.393612:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982762880 00000100:00100000:0.0:1767319229.393647:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1301 00000100:00080000:1.0:1767319229.393652:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319229.393654:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982762880:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767319229.393663:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319229.393669:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982762368:0@lo:38:kworker.0 00000100:00080000:1.0:1767319229.393681:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319229.393684:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319229.393687:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982762624:0@lo:38:kworker.0 00000100:00080000:1.0:1767319229.393696:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319229.393699:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319229.393702:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982762752:0@lo:38:kworker.0 00010000:00020000:0.0:1767319229.393739:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319229.393783:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982762880:12345-0@lo:38:kworker.0 Request processed in 128us (5419us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319229.393798:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1301 00000100:00080000:1.0:1767319229.394164:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319229.394170:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319229.394173:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982762880:0@lo:38:kworker.0 00000100:00100000:3.0:1767319229.394289:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982762240:12345-0@lo:400:kworker.0 Request processed in 2295us (13077us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319229.394296:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989 00010000:00080000:2.0:1767319229.394377:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5c00 x1853163982762240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319245 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319229.394388:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982762240:0@lo:400:kworker.0 00010000:00010000:1.0:1767319229.573209:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319230.597429:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319231.371965:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319231.372266:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613312 00000100:00100000:0.0:1767319231.372308:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613312 00000100:00100000:0.0:1767319231.372389:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 991 00000100:00100000:0.0:1767319231.372401:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915613312:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319231.372483:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915613312:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (530us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319231.372491:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 991 00000100:00080000:2.0:1767319231.373466:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319231.375015:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319231.376747:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613184 00000100:00100000:1.0:1767319231.376764:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613184 00000100:00100000:1.0:1767319231.376822:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1302 00000100:00100000:1.0:1767319231.376831:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915613184:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319231.376854:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319231.376989:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915613184:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 157us (1978us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319231.376998:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1302 00010000:00010000:0.0:1767319231.620483:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319232.644837:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319233.670015:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319233.988534:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319233.988609:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319233.988699:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763008:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319233.988725:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319233.988812:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763136 00000100:00100000:3.0:1767319233.988821:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763136 00000100:00100000:3.0:1767319233.988883:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763008 00000100:00100000:3.0:1767319233.988886:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763008 00000100:00100000:3.0:1767319233.988905:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 678 00000100:00100000:3.0:1767319233.988915:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319233.988943:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319233.988988:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319233.988995:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 678 00000100:00100000:3.0:1767319233.989013:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 679 00000100:00100000:3.0:1767319233.989019:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319233.989034:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319233.989062:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319233.989068:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 679 00000100:00080000:2.0:1767319233.994093:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319233.994104:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319233.994165:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319233.994195:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763008:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319234.500776:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319234.500787:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319234.500858:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319234.500863:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319234.500867:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319234.500875:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4477 00000100:00080000:3.0:1767319234.500881:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319234.500888:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319234.500911:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163982763392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319234.500931:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319234.500946:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982763264:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.501008:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319234.501299:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763264 00000100:00100000:1.0:1767319234.501315:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763264 00000100:00100000:1.0:1767319234.501378:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992 00000100:00100000:1.0:1767319234.501389:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982763264:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319234.501422:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319234.501450:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982763264:12345-0@lo:400:kworker.0 Request processed in 63us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319234.501457:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992 00010000:00080000:2.0:1767319234.501483:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb800 x1853163982763264/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319250 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319234.501492:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982763392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319234.501496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982763264:0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.501516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319234.501539:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763392 00000100:00100000:2.0:1767319234.501561:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763392 00000100:00100000:2.0:1767319234.501591:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1303 00000100:00100000:2.0:1767319234.501596:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982763392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319234.501612:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319234.502413:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319234.502419:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319234.502423:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319234.502428:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4477 00000100:00080000:3.0:1767319234.502432:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319234.502437:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319234.502452:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982763648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319234.502520:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319234.502533:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982763520:0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.502577:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.502613:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763520 00000100:00100000:0.0:1767319234.502618:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763520 00000100:00100000:0.0:1767319234.502660:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993 00000100:00100000:0.0:1767319234.502667:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982763520:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.502709:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982763520:12345-0@lo:400:kworker.0 Request processed in 42us (132us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319234.502716:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993 00000100:00080000:0.0:1767319234.502734:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319234.502753:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb800 x1853163982763520/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319250 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319234.502763:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982763520:0@lo:400:kworker.0 00000100:00100000:1.0:1767319234.506964:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982763648:0@lo:38:kworker.0 00000100:00100000:1.0:1767319234.507102:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.507242:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763648 00000100:00100000:0.0:1767319234.507250:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763648 00000100:00100000:0.0:1767319234.507320:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1304 00000100:00100000:0.0:1767319234.507328:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982763648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319234.507350:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319234.507395:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982763648:12345-0@lo:38:kworker.0 Request processed in 65us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319234.507403:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1304 00000100:00100000:2.0:1767319234.508244:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982763392:12345-0@lo:38:kworker.0 Request processed in 6643us (6723us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319234.508253:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1303 00000100:00080000:3.0:1767319234.508254:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319234.508262:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319234.508266:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319234.508289:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4477 00000100:00080000:3.0:1767319234.508294:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319234.508300:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319234.508332:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982763776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319234.508422:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319234.508760:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982764032:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.508810:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319234.511291:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319234.511303:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319234.511307:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982763392:0@lo:38:kworker.0 00000100:00100000:1.0:1767319234.511757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982763776:0@lo:38:kworker.0 00000100:00100000:1.0:1767319234.512622:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.512919:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763776 00000100:00100000:0.0:1767319234.512928:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763776 00000100:00100000:0.0:1767319234.512974:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 351 00000100:00100000:0.0:1767319234.512983:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982763776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319234.513645:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319234.513696:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982763776:12345-0@lo:38:kworker.0 Request processed in 712us (1076us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319234.513703:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 351 00000100:00080000:1.0:1767319234.514074:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319234.514080:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319234.514083:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982763648:0@lo:38:kworker.0 00000100:00080000:1.0:1767319234.514112:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319234.514116:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319234.514118:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982763776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319234.517834:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.517950:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319234.518095:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764032 00000100:00100000:0.0:1767319234.518104:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764032 00000100:00100000:0.0:1767319234.518156:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982763904 00000100:00100000:0.0:1767319234.518160:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982763904 00000100:00100000:0.0:1767319234.518180:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 629 00000100:00100000:0.0:1767319234.518198:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982764032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.518255:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982764032:12345-0@lo:400:kworker.0 Request processed in 67us (9444us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319234.518262:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 629 00000100:00100000:0.0:1767319234.518298:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 630 00000100:00100000:0.0:1767319234.518306:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319234.518348:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982763904:12345-0@lo:400:kworker.0 Request processed in 43us (398us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319234.518354:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 630 00000100:00100000:2.0:1767319234.518903:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982764032:0@lo:400:kworker.0 00000100:00100000:2.0:1767319234.518916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982763904:0@lo:400:kworker.0 00000100:00080000:3.0:1767319234.519433:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319234.519440:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319234.519444:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319234.519449:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4477 00000100:00080000:3.0:1767319234.519453:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319234.519458:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319234.519515:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0000 x1853163982764160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319234.519592:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982764160:0@lo:38:kworker.0 00000100:00100000:1.0:1767319234.519659:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319234.519787:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764160 00000100:00100000:3.0:1767319234.519793:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764160 00000100:00100000:3.0:1767319234.519839:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1305 00000100:00100000:3.0:1767319234.519846:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982764160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319234.520018:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319234.520074:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982764160:12345-0@lo:38:kworker.0 Request processed in 226us (414us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319234.520082:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1305 00000100:00080000:1.0:1767319234.520120:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319234.520126:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319234.520129:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982764160:0@lo:38:kworker.0 00010000:00010000:0.0:1767319234.693219:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319235.716729:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319236.491366:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319236.491414:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319236.491453:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915613696 offset 0 length 520: 4 00000100:00100000:2.0:1767319236.491518:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613440 00000100:00100000:2.0:1767319236.491529:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613440 00000100:00100000:2.0:1767319236.491582:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613568 00000100:00100000:2.0:1767319236.491585:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613568 00000100:00100000:2.0:1767319236.491651:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 994 00000100:00100000:2.0:1767319236.491660:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915613568:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319236.491738:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915613568:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (323us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319236.491746:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 994 00000100:00100000:2.0:1767319236.491762:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1306 00000100:00100000:2.0:1767319236.491767:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915613440:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319236.491788:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319236.491813:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915613440:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (459us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319236.491819:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1306 00000100:00080000:1.0:1767319236.492058:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319236.740482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319237.765354:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319238.789227:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319239.109645:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982764288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319239.109732:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319239.109815:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319239.109821:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982764416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319239.109844:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319239.109940:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764288 00000100:00100000:1.0:1767319239.109949:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764288 00000100:00100000:1.0:1767319239.110015:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764416 00000100:00100000:1.0:1767319239.110019:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764416 00000100:00100000:1.0:1767319239.110037:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 680 00000100:00100000:1.0:1767319239.110047:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982764288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319239.110102:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319239.110158:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982764288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.110166:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 680 00000100:00100000:1.0:1767319239.110184:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 681 00000100:00100000:1.0:1767319239.110189:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982764416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319239.110204:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319239.110229:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982764416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 40us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319239.110235:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 681 00000100:00100000:1.0:1767319239.110308:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982764416:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319239.110324:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982764288:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319239.110683:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319239.110687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319239.622914:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319239.622925:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319239.622988:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319239.622993:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319239.622996:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319239.623305:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319239.623323:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982764544:0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.623401:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319239.623444:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764544 00000100:00100000:2.0:1767319239.623452:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764544 00000100:00080000:3.0:1767319239.623487:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319239.623508:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995 00000100:00100000:2.0:1767319239.623519:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982764544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319239.623581:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982764544:12345-0@lo:400:kworker.0 Request processed in 64us (179us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319239.623588:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995 00010000:00080000:3.0:1767319239.623616:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fbb80 x1853163982764544/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319255 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319239.623631:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982764544:0@lo:400:kworker.0 00000100:00080000:1.0:1767319239.623648:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4483 00000100:00080000:1.0:1767319239.623652:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319239.623656:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319239.623679:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982764672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319239.623831:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319239.623835:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319239.623837:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319239.623841:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4483 00000100:00080000:1.0:1767319239.623844:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319239.623847:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319239.623859:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982764928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319239.623867:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319239.623870:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319239.623872:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319239.623875:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4483 00000100:00080000:1.0:1767319239.623877:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319239.623880:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319239.623891:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6a00 x1853163982765056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319239.623931:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319239.623934:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319239.623936:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319239.623939:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4483 00000100:00080000:1.0:1767319239.623941:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319239.623944:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319239.623954:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163982765440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319239.623977:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982765184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.624368:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982764800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.624408:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319239.624440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982765312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.624458:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764800 00000100:00100000:3.0:1767319239.624463:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764800 00000100:00100000:2.0:1767319239.624481:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319239.624496:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996 00000100:00100000:3.0:1767319239.624502:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982764800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319239.624509:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765312 00000100:00100000:2.0:1767319239.624512:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765312 00000100:00100000:2.0:1767319239.624538:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 631 00000100:00100000:3.0:1767319239.624538:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982764800:12345-0@lo:400:kworker.0 Request processed in 36us (129us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319239.624543:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982765312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.624544:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996 00000100:00080000:3.0:1767319239.624560:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319239.624575:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982765312:12345-0@lo:400:kworker.0 Request processed in 32us (94us total) trans 0 rc 0/0 00010000:00080000:3.0:1767319239.624578:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5c00 x1853163982764800/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319255 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319239.624582:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 631 00000100:00100000:3.0:1767319239.624589:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982764800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319239.624606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982765312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319239.624937:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319214/real 1767319214] req@ffff934631989500 x1853163982759040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319239 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319239.624954:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631989500 x1853163982759040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319239 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319239.624968:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982764672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319239.624996:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319239.625038:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982764928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319239.625058:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319239.625074:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982765056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319239.625097:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319239.625132:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982765440:0@lo:38:kworker.0 00000100:00100000:0.0:1767319239.625152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319239.625166:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319239.625181:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319239.625184:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982759040:0@lo:250:kworker.0 00000100:00100000:0.0:1767319239.625209:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765056 00000100:00100000:0.0:1767319239.625214:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765056 00000100:00100000:0.0:1767319239.625236:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 352 00000100:00100000:0.0:1767319239.625241:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982765056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319239.625283:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319239.625320:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982765056:12345-0@lo:38:kworker.0 Request processed in 77us (221us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319239.625326:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 352 00000100:00080000:0.0:1767319239.625354:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319239.625358:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319239.625361:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982765056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319239.625387:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764672 00000100:00100000:3.0:1767319239.625391:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764672 00000100:00080000:0.0:1767319239.625395:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319239.625399:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319239.625401:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319239.625404:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4462 00000100:00080000:0.0:1767319239.625408:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319239.625408:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982764928 00000100:00100000:3.0:1767319239.625410:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982764928 00000100:00080000:0.0:1767319239.625412:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319239.625419:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765440 00000100:00100000:3.0:1767319239.625421:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765440 00000100:00100000:0.0:1767319239.625428:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903100 x1853163982765568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319239.625429:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1307 00000100:00100000:3.0:1767319239.625434:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982764672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319239.625444:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319239.625455:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982765568:0@lo:250:kworker.0 00000400:00000100:0.0:1767319239.625470:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982765568 offset 0 length 520: 4 00000100:00100000:3.0:1767319239.625470:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982764672:12345-0@lo:38:kworker.0 Request processed in 35us (473us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319239.625475:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1307 00000100:00100000:3.0:1767319239.625487:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1308 00000100:00080000:0.0:1767319239.625490:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319239.625491:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982764928:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767319239.625493:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319239.625495:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982764672:0@lo:38:kworker.0 00010000:00020000:3.0:1767319239.625499:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.625524:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982764928:12345-0@lo:38:kworker.0 Request processed in 31us (464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319239.625529:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1308 00000100:00080000:0.0:1767319239.625538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319239.625540:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1309 00000100:00080000:0.0:1767319239.625542:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319239.625544:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982765440:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319239.625545:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982764928:0@lo:38:kworker.0 00010000:00020000:3.0:1767319239.625553:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319239.625590:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982765440:12345-0@lo:38:kworker.0 Request processed in 44us (436us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319239.625596:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1309 00000100:00100000:1.0:1767319239.625609:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319239.625628:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765184 00000100:00100000:3.0:1767319239.625632:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765184 00000100:00100000:3.0:1767319239.625674:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 632 00000100:00100000:3.0:1767319239.625680:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982765184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319239.625715:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982765184:12345-0@lo:400:kworker.0 Request processed in 35us (106us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319239.625721:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 632 00000100:00080000:0.0:1767319239.626084:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319239.626089:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319239.626092:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982765440:0@lo:38:kworker.0 00000100:00100000:1.0:1767319239.626326:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982765184:0@lo:400:kworker.0 00010000:00010000:1.0:1767319239.814301:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319240.838808:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319241.616786:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319241.619748:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613952 00000100:00100000:1.0:1767319241.619765:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613952 00000100:00100000:3.0:1767319241.619775:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767319241.619926:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319241.624461:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915613824 00000100:00100000:2.0:1767319241.624479:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915613824 00000100:00100000:2.0:1767319241.624548:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 997 00000100:00100000:2.0:1767319241.624565:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915613952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319241.624666:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915613952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 109us (7889us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319241.624674:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 997 00000100:00100000:2.0:1767319241.624696:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1310 00000100:00100000:2.0:1767319241.624701:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915613824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319241.624721:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319241.624746:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915613824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (4977us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319241.624754:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1310 00010000:00010000:1.0:1767319241.861721:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319242.885134:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319243.908614:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319244.230606:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982765696:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319244.230691:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319244.231371:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765696 00000100:00100000:2.0:1767319244.231380:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765696 00000100:00100000:2.0:1767319244.231461:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 682 00000100:00100000:2.0:1767319244.231471:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982765696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319244.231504:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319244.231564:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982765696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (873us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319244.231572:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 682 00000100:00100000:2.0:1767319244.231623:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982765696:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319244.231687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319244.234876:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319244.239806:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982765824:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319244.239972:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319244.240113:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765824 00000100:00100000:3.0:1767319244.240123:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765824 00000100:00100000:3.0:1767319244.240210:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00100000:3.0:1767319244.240221:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982765824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319244.240255:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319244.240359:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982765824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 139us (385us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319244.240367:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00000100:00100000:0.0:1767319244.240421:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982765824:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319244.240476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319244.740544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319244.740558:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319244.740638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319244.740643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319244.740646:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319244.740656:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4488 00000100:00080000:1.0:1767319244.740660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319244.740668:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319244.740693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6a00 x1853163982766080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319244.740750:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982766080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319244.740821:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319244.740892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319244.740895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319244.740897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319244.740901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4488 00000100:00080000:1.0:1767319244.740904:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319244.740908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319244.740920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913b80 x1853163982766336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319244.740981:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982765952:0@lo:400:kworker.0 00000100:00100000:2.0:1767319244.741014:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319244.741036:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319244.741042:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982766208:0@lo:400:kworker.0 00000100:00100000:2.0:1767319244.741059:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319244.741114:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766080 00000100:00100000:2.0:1767319244.741122:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766080 00000100:00100000:2.0:1767319244.741150:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982765952 00000100:00100000:2.0:1767319244.741153:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982765952 00000100:00100000:2.0:1767319244.741200:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766208 00000100:00100000:2.0:1767319244.741203:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766208 00000100:00100000:2.0:1767319244.741222:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998 00000100:00100000:2.0:1767319244.741240:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982765952:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319244.741335:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982765952:12345-0@lo:400:kworker.0 Request processed in 105us (320us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319244.741342:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998 00000100:00100000:2.0:1767319244.741359:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999 00000100:00100000:2.0:1767319244.741364:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982766208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319244.741391:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982766208:12345-0@lo:400:kworker.0 Request processed in 27us (331us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319244.741396:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999 00000100:00100000:2.0:1767319244.741408:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1311 00000100:00100000:2.0:1767319244.741412:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982766080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319244.741431:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319244.741545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319244.741582:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7b80 x1853163982766208/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319260 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319244.741595:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982766208:0@lo:400:kworker.0 00010000:00080000:3.0:1767319244.741609:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4e00 x1853163982765952/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319260 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319244.741617:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982765952:0@lo:400:kworker.0 00000100:00100000:0.0:1767319244.742070:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982766336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319244.742108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319244.742184:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766336 00000100:00100000:3.0:1767319244.742189:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766336 00000100:00100000:3.0:1767319244.742215:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1312 00000100:00100000:3.0:1767319244.742220:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982766336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319244.742242:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319244.742299:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982766336:12345-0@lo:38:kworker.0 Request processed in 77us (190us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319244.742305:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1312 00000100:00080000:1.0:1767319244.742454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319244.742459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319244.742462:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319244.742466:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4488 00000100:00080000:1.0:1767319244.742469:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319244.742473:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319244.742495:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911500 x1853163982766464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319244.742540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319244.742543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319244.742545:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319244.742548:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4488 00000100:00080000:1.0:1767319244.742550:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319244.742554:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319244.742563:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163982766848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319244.742589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982766720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319244.742617:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319244.742640:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319244.742644:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982766592:0@lo:400:kworker.0 00000100:00100000:1.0:1767319244.742660:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319244.742680:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766720 00000100:00100000:1.0:1767319244.742684:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766720 00000100:00100000:1.0:1767319244.742710:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766592 00000100:00100000:1.0:1767319244.742713:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766592 00000100:00100000:1.0:1767319244.742730:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 633 00000100:00100000:1.0:1767319244.742735:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982766720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319244.742768:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982766720:12345-0@lo:400:kworker.0 Request processed in 33us (150us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319244.742773:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 633 00000100:00100000:1.0:1767319244.742785:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 634 00000100:00100000:1.0:1767319244.742790:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982766592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319244.742810:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982766592:12345-0@lo:400:kworker.0 Request processed in 20us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319244.742815:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 634 00000100:00100000:1.0:1767319244.743047:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982766592:0@lo:400:kworker.0 00000100:00100000:1.0:1767319244.743116:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982766720:0@lo:400:kworker.0 00000100:00100000:2.0:1767319244.743203:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982766080:12345-0@lo:38:kworker.0 Request processed in 1788us (2381us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319244.743211:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1311 00000100:00080000:0.0:1767319244.743461:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319244.743466:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319244.743470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982766080:0@lo:38:kworker.0 00000100:00100000:0.0:1767319244.743486:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982766464:0@lo:38:kworker.0 00000100:00100000:0.0:1767319244.743516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319244.743579:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982766848:0@lo:38:kworker.0 00000100:00100000:0.0:1767319244.743600:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319244.743661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319244.743665:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319244.743668:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982766336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319244.743711:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766464 00000100:00100000:2.0:1767319244.743715:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766464 00000100:00100000:2.0:1767319244.743771:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 353 00000100:00100000:2.0:1767319244.743777:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982766464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319244.743793:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319244.743833:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982766464:12345-0@lo:38:kworker.0 Request processed in 56us (316us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319244.743839:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 353 00000100:00080000:1.0:1767319244.743944:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319244.743949:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319244.743952:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982766464:0@lo:38:kworker.0 00000100:00100000:3.0:1767319244.744010:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766848 00000100:00100000:3.0:1767319244.744014:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766848 00000100:00100000:3.0:1767319244.744042:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1313 00000100:00100000:3.0:1767319244.744046:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982766848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319244.744056:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319244.744092:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982766848:12345-0@lo:38:kworker.0 Request processed in 44us (490us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319244.744098:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1313 00000100:00080000:1.0:1767319244.744353:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319244.744358:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319244.744361:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982766848:0@lo:38:kworker.0 00010000:00010000:1.0:1767319244.932729:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319245.956945:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319246.731524:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319246.731663:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614208 00000100:00100000:2.0:1767319246.731674:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614208 00000100:00100000:2.0:1767319246.731760:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1000 00000100:00100000:2.0:1767319246.731770:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915614208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319246.731854:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915614208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (339us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319246.731862:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1000 00000100:00080000:3.0:1767319246.732079:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319246.733178:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319246.737592:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614080 00000100:00100000:0.0:1767319246.737609:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614080 00000100:00100000:0.0:1767319246.738135:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1314 00000100:00100000:0.0:1767319246.738146:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915614080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319246.738169:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319246.738258:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915614080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (5074us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319246.738265:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1314 00010000:00010000:0.0:1767319246.980435:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319248.011410:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319249.028557:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319249.348557:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982766976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319249.348557:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982767104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319249.348634:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.348750:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982766976 00000100:00100000:2.0:1767319249.348759:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982766976 00000100:00100000:2.0:1767319249.348840:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 684 00000100:00100000:2.0:1767319249.348849:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982766976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319249.348882:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319249.348929:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982766976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319249.348936:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 684 00000100:00080000:0.0:1767319249.349003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319249.349038:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982766976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319249.349761:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.349845:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767104 00000100:00100000:2.0:1767319249.349851:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767104 00000100:00100000:2.0:1767319249.349899:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 685 00000100:00100000:2.0:1767319249.349906:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982767104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319249.349930:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767319249.349999:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319249.352949:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982767104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3035us (3181us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319249.352960:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982767104:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319249.352963:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 685 00000100:00080000:1.0:1767319249.860465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319249.860478:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319249.860554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319249.860559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319249.860563:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319249.860571:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4493 00000100:00080000:1.0:1767319249.860575:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319249.860581:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319249.860602:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581500 x1853163982767360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319249.860623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319249.860626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319249.860628:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319249.860631:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4493 00000100:00080000:1.0:1767319249.860634:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319249.860637:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319249.860652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163982767616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319249.860660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319249.860663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319249.860665:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319249.860668:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4493 00000100:00080000:1.0:1767319249.860671:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319249.860675:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319249.860687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982767744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319249.860721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319249.860724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319249.860725:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319249.860728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4493 00000100:00080000:1.0:1767319249.860731:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319249.860734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319249.860743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff480 x1853163982768128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319249.860775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982768000:0@lo:400:kworker.0 00000100:00100000:1.0:1767319249.860843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319249.860884:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319249.860890:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982767872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319249.860911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.862381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982767360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319249.862450:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.862478:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982767616:0@lo:38:kworker.0 00000100:00100000:2.0:1767319249.862500:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.862517:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982767744:0@lo:38:kworker.0 00000100:00100000:2.0:1767319249.862541:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.862580:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982768128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319249.862602:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319249.862633:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767360 00000100:00100000:2.0:1767319249.862641:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767360 00000100:00100000:2.0:1767319249.862672:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767616 00000100:00100000:2.0:1767319249.862675:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767616 00000100:00100000:2.0:1767319249.862686:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768128 00000100:00100000:2.0:1767319249.862688:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768128 00000100:00100000:2.0:1767319249.862702:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1315 00000100:00100000:2.0:1767319249.862710:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982767360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319249.862733:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319249.862776:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982767360:12345-0@lo:38:kworker.0 Request processed in 65us (324us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319249.862782:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1315 00000100:00100000:2.0:1767319249.862798:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1316 00000100:00100000:2.0:1767319249.862801:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982767616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319249.862810:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319249.862832:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982767616:12345-0@lo:38:kworker.0 Request processed in 28us (331us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319249.862837:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1316 00000100:00100000:2.0:1767319249.862848:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1317 00000100:00100000:2.0:1767319249.862852:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982768128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319249.862862:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319249.862883:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982768128:12345-0@lo:38:kworker.0 Request processed in 30us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319249.862888:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1317 00000100:00100000:2.0:1767319249.862905:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767744 00000100:00100000:2.0:1767319249.862909:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767744 00000100:00100000:2.0:1767319249.862925:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 354 00000100:00100000:2.0:1767319249.862930:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982767744:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319249.862940:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319249.862963:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982767744:12345-0@lo:38:kworker.0 Request processed in 31us (421us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319249.862969:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 354 00000100:00080000:2.0:1767319249.863001:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319249.863006:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319249.863010:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982767360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319249.863018:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319249.863021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319249.863024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982767616:0@lo:38:kworker.0 00000100:00080000:2.0:1767319249.863031:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319249.863035:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319249.863038:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982767744:0@lo:38:kworker.0 00000100:00080000:2.0:1767319249.863044:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319249.863047:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319249.863050:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982768128:0@lo:38:kworker.0 00000100:00100000:3.0:1767319249.865549:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768000 00000100:00100000:3.0:1767319249.865567:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768000 00000100:00100000:3.0:1767319249.865633:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767872 00000100:00100000:3.0:1767319249.865636:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767872 00000100:00100000:3.0:1767319249.865673:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 635 00000100:00100000:1.0:1767319249.865681:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 636 00000100:00100000:3.0:1767319249.865682:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982768000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319249.865690:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982767872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319249.865804:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982767488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319249.865918:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982767232:0@lo:400:kworker.0 00000100:00100000:3.0:1767319249.868825:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982768000:12345-0@lo:400:kworker.0 Request processed in 3136us (7973us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319249.868839:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 635 00000100:00100000:3.0:1767319249.868922:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982768000:0@lo:400:kworker.0 00000100:00100000:1.0:1767319249.869043:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982767872:12345-0@lo:400:kworker.0 Request processed in 3353us (8130us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319249.869051:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 636 00000100:00100000:3.0:1767319249.871580:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982767872:0@lo:400:kworker.0 00000100:00100000:0.0:1767319249.871710:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319249.875242:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767488 00000100:00100000:3.0:1767319249.875257:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767488 00000100:00100000:3.0:1767319249.875386:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001 00000100:00100000:3.0:1767319249.875400:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982767488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319249.875505:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982767488:12345-0@lo:400:kworker.0 Request processed in 110us (3794us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319249.875512:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001 00000100:00080000:0.0:1767319249.875601:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319249.875641:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583480 x1853163982767488/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319265 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319249.875656:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982767488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319249.876623:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319249.876682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982767232 00000100:00100000:1.0:1767319249.876690:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982767232 00000100:00100000:1.0:1767319249.876753:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1002 00000100:00100000:1.0:1767319249.876761:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982767232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319249.876819:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982767232:12345-0@lo:400:kworker.0 Request processed in 59us (199us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319249.876826:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1002 00010000:00080000:2.0:1767319249.876887:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581f80 x1853163982767232/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319265 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319249.876898:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982767232:0@lo:400:kworker.0 00000100:00080000:0.0:1767319249.876933:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319250.052431:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319251.076591:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319251.851830:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319251.852059:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614464 00000100:00100000:0.0:1767319251.852074:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614464 00000100:00100000:0.0:1767319251.852184:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1003 00000100:00100000:0.0:1767319251.852197:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915614464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319251.852367:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915614464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 172us (552us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319251.852379:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1003 00000100:00080000:3.0:1767319251.852983:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319251.855995:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319251.856046:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614336 00000100:00100000:1.0:1767319251.856066:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614336 00000100:00100000:1.0:1767319251.856139:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1318 00000100:00100000:1.0:1767319251.856149:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915614336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319251.856720:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319251.856825:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915614336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 675us (842us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319251.856836:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1318 00010000:00010000:1.0:1767319252.100651:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319253.127010:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319254.148508:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319254.468558:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982768256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319254.468653:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319254.468752:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319254.468760:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982768384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319254.468772:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768256 00000100:00100000:0.0:1767319254.468784:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768256 00000100:00100000:3.0:1767319254.468817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319254.468861:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768384 00000100:00100000:0.0:1767319254.468865:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768384 00000100:00080000:2.0:1767319254.468917:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319254.468923:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319254.473550:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00100000:3.0:1767319254.475089:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982768256:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319254.475116:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 687 00000100:00100000:0.0:1767319254.475135:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982768384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319254.475152:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:0.0:1767319254.475161:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319254.479340:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982768256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5755us (10669us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319254.479364:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00100000:2.0:1767319254.479586:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982768256:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319254.487799:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982768384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 12657us (18966us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319254.487818:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 687 00000100:00100000:2.0:1767319254.488022:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982768384:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319254.980556:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319254.980570:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319254.980701:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982768512:0@lo:400:kworker.0 00000100:00100000:0.0:1767319254.980805:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319254.980912:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319254.980918:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319254.980923:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319254.980930:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4498 00000100:00080000:3.0:1767319254.980935:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319254.980943:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319254.980973:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ed80 x1853163982768640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319254.981050:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768512 00000100:00100000:1.0:1767319254.981061:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768512 00000100:00100000:1.0:1767319254.981144:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004 00000100:00100000:1.0:1767319254.981156:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982768512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319254.981517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982768768:0@lo:400:kworker.0 00000100:00100000:2.0:1767319254.981558:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319254.981596:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768768 00000100:00100000:2.0:1767319254.981599:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768768 00000100:00100000:2.0:1767319254.981625:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005 00000100:00100000:2.0:1767319254.981629:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982768768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319254.981656:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982768768:12345-0@lo:400:kworker.0 Request processed in 26us (99us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319254.981662:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005 00010000:00080000:2.0:1767319254.981698:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d880 x1853163982768768/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319270 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319254.981714:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982768768:0@lo:400:kworker.0 00000100:00100000:2.0:1767319254.981744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982768640:0@lo:38:kworker.0 00000100:00100000:2.0:1767319254.981761:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319254.981788:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768640 00000100:00100000:2.0:1767319254.981792:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768640 00000100:00100000:2.0:1767319254.981811:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1319 00000100:00100000:2.0:1767319254.981815:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982768640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319254.981838:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319254.981873:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982768640:12345-0@lo:38:kworker.0 Request processed in 56us (109us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319254.981880:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1319 00000100:00080000:2.0:1767319254.981904:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319254.981909:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319254.981913:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982768640:0@lo:38:kworker.0 00000100:00080000:2.0:1767319254.981953:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319254.984422:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319254.984435:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319254.984440:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319254.984449:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4498 00000100:00080000:3.0:1767319254.984455:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319254.984462:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319254.984511:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e300 x1853163982768896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319254.984541:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319254.984545:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319254.984547:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319254.984550:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4498 00000100:00080000:3.0:1767319254.984554:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319254.984557:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319254.984570:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f480 x1853163982769024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319254.984580:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319254.984583:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319254.984585:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319254.984589:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4498 00000100:00080000:3.0:1767319254.984591:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319254.984594:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319254.984604:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d500 x1853163982769152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319254.984674:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982768896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319254.984754:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319254.984805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982769024:0@lo:38:kworker.0 00000100:00100000:3.0:1767319254.984832:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319254.984855:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982769152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319254.984878:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319254.984920:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769024 00000100:00100000:3.0:1767319254.984929:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769024 00000100:00100000:3.0:1767319254.984970:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 355 00000100:00100000:3.0:1767319254.984978:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982769024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319254.985004:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319254.985056:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982769024:12345-0@lo:38:kworker.0 Request processed in 77us (221us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319254.985064:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 355 00000100:00100000:3.0:1767319254.985104:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982768896 00000100:00100000:3.0:1767319254.985109:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982768896 00000100:00100000:3.0:1767319254.985132:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769152 00000100:00100000:3.0:1767319254.985135:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769152 00000100:00100000:3.0:1767319254.985154:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1320 00000100:00100000:3.0:1767319254.985158:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982768896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319254.985168:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319254.985194:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982768896:12345-0@lo:38:kworker.0 Request processed in 34us (440us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319254.985200:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1320 00000100:00100000:3.0:1767319254.985212:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1321 00000100:00100000:3.0:1767319254.985216:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982769152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319254.985225:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319254.985247:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982769152:12345-0@lo:38:kworker.0 Request processed in 30us (369us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319254.985253:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1321 00000100:00080000:3.0:1767319254.985318:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319254.985323:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319254.985326:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982768896:0@lo:38:kworker.0 00000100:00080000:3.0:1767319254.985335:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319254.985338:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319254.985341:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982769024:0@lo:38:kworker.0 00000100:00080000:3.0:1767319254.985348:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319254.985351:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319254.985355:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982769152:0@lo:38:kworker.0 00000100:00100000:1.0:1767319254.985478:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982768512:12345-0@lo:400:kworker.0 Request processed in 4324us (4673us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319254.985485:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004 00010000:00080000:0.0:1767319254.986350:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d180 x1853163982768512/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319270 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319254.986366:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982768512:0@lo:400:kworker.0 00010000:00010000:1.0:1767319255.173153:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319256.197957:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319256.980027:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319256.980095:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319256.980956:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614720 00000100:00100000:3.0:1767319256.981019:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614720 00000100:00100000:3.0:1767319256.981189:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614592 00000100:00100000:3.0:1767319256.981193:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614592 00000100:00100000:3.0:1767319256.981232:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1006 00000100:00100000:3.0:1767319256.981245:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915614720:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319256.981370:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915614720:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 128us (1357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319256.981390:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1006 00000100:00100000:3.0:1767319256.981415:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1322 00000100:00100000:3.0:1767319256.981420:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915614592:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319256.981455:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319256.981484:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915614592:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (1389us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319256.981491:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1322 00000100:00080000:2.0:1767319256.982611:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319257.220668:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319258.244497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319259.268627:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319259.589090:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982769280:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319259.589234:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319259.589355:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319259.589362:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982769408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319259.589390:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319259.589517:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769280 00000100:00100000:3.0:1767319259.589528:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769280 00000100:00100000:3.0:1767319259.589604:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769408 00000100:00100000:3.0:1767319259.589608:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769408 00000100:00100000:3.0:1767319259.589633:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 688 00000100:00100000:3.0:1767319259.589643:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982769280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319259.589682:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319259.589736:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982769280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (502us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319259.589745:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 688 00000100:00100000:1.0:1767319259.589757:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 689 00000100:00100000:1.0:1767319259.589767:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934602e4fc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982769408:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319259.590324:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319259.590332:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319259.590390:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982769280:0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319259.593976:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319259.594119:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934602e4fc50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982769408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4350us (4723us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319259.594130:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 689 00000100:00100000:0.0:1767319259.596140:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982769408:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319260.100995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319260.101015:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319260.101090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319260.101094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319260.101097:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319260.101106:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4503 00000100:00080000:1.0:1767319260.101112:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319260.101118:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319260.101149:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163982769664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319260.101186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319260.101189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319260.101191:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319260.101195:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4503 00000100:00080000:1.0:1767319260.101197:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319260.101202:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319260.101213:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff480 x1853163982769920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319260.101222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319260.101225:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319260.101227:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319260.101231:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4503 00000100:00080000:1.0:1767319260.101234:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319260.101237:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319260.101250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982770048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319260.101258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319260.101262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319260.101263:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319260.101267:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4503 00000100:00080000:1.0:1767319260.102681:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319260.102694:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319260.102733:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583b80 x1853163982770176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319260.103087:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982769664:0@lo:38:kworker.0 00000100:00100000:3.0:1767319260.103169:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.103245:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982769920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319260.103288:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.103591:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982770048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319260.103623:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.103648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982770176:0@lo:38:kworker.0 00000100:00100000:3.0:1767319260.103669:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.103709:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770048 00000100:00100000:3.0:1767319260.103717:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770048 00000100:00100000:0.0:1767319260.104710:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769664 00000100:00100000:0.0:1767319260.104717:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769664 00000100:00100000:0.0:1767319260.104786:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769920 00000100:00100000:0.0:1767319260.104789:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769920 00000100:00100000:0.0:1767319260.104810:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770176 00000100:00100000:0.0:1767319260.104813:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770176 00000100:00100000:0.0:1767319260.104831:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1323 00000100:00100000:0.0:1767319260.104839:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982769664:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319260.104861:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319260.104921:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982769664:12345-0@lo:38:kworker.0 Request processed in 82us (1752us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319260.104929:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1323 00000100:00100000:0.0:1767319260.104947:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1324 00000100:00100000:0.0:1767319260.104951:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982769920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319260.104961:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319260.104986:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982769920:12345-0@lo:38:kworker.0 Request processed in 33us (1717us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319260.104992:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1324 00000100:00100000:0.0:1767319260.105014:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1325 00000100:00100000:0.0:1767319260.105019:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982770176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319260.105028:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319260.105052:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982770176:12345-0@lo:38:kworker.0 Request processed in 32us (1382us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319260.105061:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1325 00000100:00080000:0.0:1767319260.105108:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319260.105114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319260.105118:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982769664:0@lo:38:kworker.0 00000100:00080000:0.0:1767319260.105138:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319260.105142:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319260.105144:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982769920:0@lo:38:kworker.0 00000100:00080000:0.0:1767319260.105152:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319260.105154:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319260.105157:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982770176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319260.105193:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982769536:0@lo:400:kworker.0 00000100:00100000:0.0:1767319260.105223:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319260.105246:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319260.105253:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982769792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319260.105322:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319260.105820:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 356 00000100:00100000:3.0:1767319260.105827:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982770048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319260.105839:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319260.105881:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982770048:12345-0@lo:38:kworker.0 Request processed in 54us (2257us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319260.105887:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 356 00000100:00100000:3.0:1767319260.105908:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769536 00000100:00100000:3.0:1767319260.105913:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769536 00000100:00100000:3.0:1767319260.105946:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982769792 00000100:00100000:3.0:1767319260.105949:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982769792 00000100:00100000:3.0:1767319260.105964:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007 00000100:00100000:3.0:1767319260.105969:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982769536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.106020:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982769536:12345-0@lo:400:kworker.0 Request processed in 51us (796us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319260.106027:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007 00000100:00100000:3.0:1767319260.106038:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008 00000100:00100000:3.0:1767319260.106046:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982769792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319260.106073:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982769792:12345-0@lo:400:kworker.0 Request processed in 28us (762us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319260.106079:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008 00000100:00080000:2.0:1767319260.106250:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319260.106255:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319260.106259:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982770048:0@lo:38:kworker.0 00000100:00080000:2.0:1767319260.106301:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319260.107881:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff800 x1853163982769792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319276 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319260.107899:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982769792:0@lo:400:kworker.0 00010000:00080000:0.0:1767319260.107913:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163982769536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319276 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319260.107922:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982769536:0@lo:400:kworker.0 00010000:00010000:1.0:1767319260.293982:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319261.076448:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319261.076500:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915614848 offset 0 length 520: 4 00000100:00100000:3.0:1767319261.076551:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319261.076682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615104 00000100:00100000:0.0:1767319261.076694:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615104 00000100:00100000:0.0:1767319261.076763:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915614976 00000100:00100000:0.0:1767319261.076767:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915614976 00000100:00100000:0.0:1767319261.076790:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1009 00000100:00100000:0.0:1767319261.076805:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319261.076881:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (455us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319261.076889:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1009 00000100:00100000:0.0:1767319261.076908:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1326 00000100:00100000:0.0:1767319261.076913:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915614976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319261.076933:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319261.076960:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915614976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (409us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319261.076966:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1326 00000100:00080000:3.0:1767319261.078265:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319261.318336:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319262.340865:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319263.368294:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319264.196442:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319239/real 1767319239] req@ffff934631903100 x1853163982765568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319264 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319264.196474:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631903100 x1853163982765568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319264 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319264.196491:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319264.196553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319264.196559:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982765568:0@lo:250:kworker.0 00000100:00080000:1.0:1767319264.198325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319264.198332:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319264.198335:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319264.198340:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4488 00000100:00080000:1.0:1767319264.198353:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319264.198359:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319264.198388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982770304/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319264.198722:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982770304:0@lo:250:kworker.0 00000400:00000100:2.0:1767319264.198778:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982770304 offset 0 length 520: 4 00000100:00080000:1.0:1767319264.198875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319264.198880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319264.198882:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319264.198885:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4508 00000100:00080000:1.0:1767319264.198888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319264.198892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319264.198908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375f80 x1853163982770560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319264.198917:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982770432:0@lo:400:kworker.0 00000100:00100000:0.0:1767319264.198967:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319264.199061:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770432 00000100:00100000:3.0:1767319264.199070:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770432 00000100:00100000:3.0:1767319264.199140:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1010 00000100:00100000:3.0:1767319264.199150:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982770432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319264.199202:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982770432:12345-0@lo:400:kworker.0 Request processed in 54us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319264.199209:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1010 00000100:00100000:2.0:1767319264.199233:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982770560:0@lo:38:kworker.0 00000100:00100000:2.0:1767319264.199261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319264.199293:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770560 00000100:00100000:3.0:1767319264.199298:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770560 00000100:00100000:3.0:1767319264.199318:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1327 00000100:00100000:3.0:1767319264.199322:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982770560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319264.199342:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319264.199376:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982770560:12345-0@lo:38:kworker.0 Request processed in 52us (114us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319264.199384:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1327 00000100:00080000:2.0:1767319264.199407:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319264.199422:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319264.199426:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982770560:0@lo:38:kworker.0 00000100:00080000:0.0:1767319264.199447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319264.199468:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583b80 x1853163982770432/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319280 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319264.199480:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982770432:0@lo:400:kworker.0 00000100:00080000:1.0:1767319264.199646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319264.199650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319264.199652:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319264.199656:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4508 00000100:00080000:1.0:1767319264.199659:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319264.199664:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319264.199677:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163982770816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319264.199677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982770688:0@lo:400:kworker.0 00000100:00080000:1.0:1767319264.199687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319264.199690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319264.199691:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319264.199695:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4508 00000100:00080000:1.0:1767319264.199697:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319264.199704:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319264.199838:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319264.199880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982770816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319264.199904:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319264.199955:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770688 00000100:00100000:2.0:1767319264.199960:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770688 00000100:00100000:2.0:1767319264.199989:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770816 00000100:00100000:2.0:1767319264.199991:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770816 00000100:00100000:2.0:1767319264.200017:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011 00000100:00100000:2.0:1767319264.200024:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982770688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319264.200056:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982770688:12345-0@lo:400:kworker.0 Request processed in 33us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319264.200062:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011 00010000:00080000:3.0:1767319264.200065:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375f80 x1853163982770688/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319280 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319264.200076:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982770688:0@lo:400:kworker.0 00000100:00100000:2.0:1767319264.200086:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1328 00000100:00100000:2.0:1767319264.200090:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982770816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319264.200102:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319264.200133:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982770816:12345-0@lo:38:kworker.0 Request processed in 41us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319264.200139:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1328 00000100:00080000:3.0:1767319264.200161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319264.200165:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319264.200168:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982770816:0@lo:38:kworker.0 00000100:00080000:0.0:1767319264.201642:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319264.202127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163982770944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319264.202199:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982770944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319264.202200:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982771072:0@lo:400:kworker.0 00000100:00080000:1.0:1767319264.202286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319264.202290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319264.202292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319264.202296:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4508 00000100:00080000:1.0:1767319264.202299:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319264.202304:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319264.202354:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982771200:0@lo:400:kworker.0 00000100:00100000:1.0:1767319264.202387:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163982771328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319264.203821:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319264.203882:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982771328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319264.203921:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319264.203993:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771328 00000100:00100000:3.0:1767319264.204002:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771328 00000100:00100000:3.0:1767319264.204041:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1329 00000100:00100000:3.0:1767319264.204048:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982771328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319264.204068:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319264.204114:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982771328:12345-0@lo:38:kworker.0 Request processed in 65us (192us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319264.204121:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1329 00000100:00080000:3.0:1767319264.204161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319264.204166:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319264.204169:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982771328:0@lo:38:kworker.0 00000100:00100000:1.0:1767319264.207897:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982770944 00000100:00100000:1.0:1767319264.207913:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982770944 00000100:00100000:1.0:1767319264.207980:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 357 00000100:00100000:1.0:1767319264.207988:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982770944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319264.208020:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319264.208112:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982770944:12345-0@lo:38:kworker.0 Request processed in 124us (4296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319264.208119:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 357 00000100:00080000:3.0:1767319264.208144:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319264.208149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319264.208153:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982770944:0@lo:38:kworker.0 00000100:00100000:0.0:1767319264.208976:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.209172:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771072 00000100:00100000:1.0:1767319264.209177:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771072 00000100:00100000:1.0:1767319264.209221:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 637 00000100:00100000:1.0:1767319264.209235:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982771072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319264.209313:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982771072:12345-0@lo:400:kworker.0 Request processed in 85us (338us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319264.209321:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 637 00000100:00100000:3.0:1767319264.209335:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982771072:0@lo:400:kworker.0 00000100:00080000:0.0:1767319264.209428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319264.209479:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.209533:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771200 00000100:00100000:1.0:1767319264.209536:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771200 00000100:00100000:1.0:1767319264.209608:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 638 00000100:00100000:1.0:1767319264.209615:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982771200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319264.209656:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982771200:12345-0@lo:400:kworker.0 Request processed in 42us (175us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319264.209662:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 638 00000100:00100000:2.0:1767319264.211062:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982771200:0@lo:400:kworker.0 00000100:00080000:0.0:1767319264.211127:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319264.388448:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319264.708841:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319264.708862:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982771456:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319264.708941:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319264.709101:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771456 00000100:00100000:2.0:1767319264.709109:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771456 00000100:00100000:2.0:1767319264.709167:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 690 00000100:00100000:2.0:1767319264.709175:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982771456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319264.709202:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319264.709247:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982771456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319264.709254:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 690 00000100:00100000:0.0:1767319264.709562:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319264.709573:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982771584:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319264.709601:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319264.709675:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771584 00000100:00100000:1.0:1767319264.709680:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771584 00000100:00100000:1.0:1767319264.709709:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 691 00000100:00100000:1.0:1767319264.709716:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982771584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319264.709732:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319264.709768:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982771584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319264.709774:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 691 00000100:00100000:1.0:1767319264.709813:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982771456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319264.709915:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982771584:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319265.412585:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319266.197670:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319266.197719:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319266.197790:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615360 00000100:00100000:1.0:1767319266.197800:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615360 00000100:00100000:1.0:1767319266.197881:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615232 00000100:00100000:1.0:1767319266.197885:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615232 00000100:00100000:1.0:1767319266.197917:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1012 00000100:00100000:1.0:1767319266.197928:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319266.197936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319266.197988:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1330 00000100:00100000:2.0:1767319266.197996:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915615232:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:1.0:1767319266.198011:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (350us total) trans 0 rc 0/0 00010000:00020000:2.0:1767319266.198021:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319266.198026:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1012 00000100:00100000:2.0:1767319266.198060:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915615232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (339us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319266.198068:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1330 00010000:00010000:1.0:1767319266.436663:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319267.460493:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319268.484723:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319269.316380:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319269.316392:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319269.316458:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319269.316462:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319269.316465:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319269.316474:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4512 00000100:00080000:1.0:1767319269.316489:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319269.316496:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982771712:0@lo:400:kworker.0 00000100:00080000:1.0:1767319269.316498:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319269.316527:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163982771840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319269.316566:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319269.316569:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319269.316573:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319269.316574:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319269.316580:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4512 00000100:00080000:1.0:1767319269.316583:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319269.316587:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319269.316601:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163982772096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319269.316609:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319269.316613:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319269.316615:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319269.316618:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4512 00000100:00080000:1.0:1767319269.316620:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319269.316624:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319269.316627:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771712 00000100:00100000:3.0:1767319269.316635:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771712 00000100:00100000:1.0:1767319269.316639:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1f80 x1853163982772224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319269.316678:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319269.316682:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319269.316684:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319269.316688:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4512 00000100:00080000:1.0:1767319269.316691:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319269.316694:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013 00000100:00080000:1.0:1767319269.316698:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319269.316707:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982771712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319269.316714:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903100 x1853163982772608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319269.316737:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982772352:0@lo:400:kworker.0 00000100:00100000:3.0:1767319269.316757:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982771712:12345-0@lo:400:kworker.0 Request processed in 52us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319269.316765:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013 00000100:00100000:1.0:1767319269.316786:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319269.316817:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319269.316824:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982772480:0@lo:400:kworker.0 00010000:00080000:3.0:1767319269.316826:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddc00 x1853163982771712/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319285 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319269.316854:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982771712:0@lo:400:kworker.0 00000100:00100000:1.0:1767319269.316862:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319269.316872:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319269.316889:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772352 00000100:00100000:3.0:1767319269.316894:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772352 00000100:00100000:1.0:1767319269.316901:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772480 00000100:00100000:1.0:1767319269.316907:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772480 00000100:00100000:3.0:1767319269.316923:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 639 00000100:00100000:3.0:1767319269.316932:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982772352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319269.316966:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982772352:12345-0@lo:400:kworker.0 Request processed in 36us (181us total) trans 0 rc 0/0 00000100:00080000:2.0:1767319269.317256:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319269.317262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319269.317265:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319269.317316:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982771968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319269.317349:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319269.317381:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771968 00000100:00100000:2.0:1767319269.317385:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771968 00000100:00100000:2.0:1767319269.317417:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1014 00000100:00100000:2.0:1767319269.317424:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982771968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319269.317459:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982771968:12345-0@lo:400:kworker.0 Request processed in 36us (109us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319269.317465:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1014 00000100:00080000:2.0:1767319269.317491:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319269.317509:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ded80 x1853163982771968/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319285 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319269.317519:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982771968:0@lo:400:kworker.0 00000100:00100000:3.0:1767319269.318113:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 639 00000100:00100000:2.0:1767319269.318130:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 640 00000100:00100000:2.0:1767319269.318142:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982772480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319269.318187:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982772480:12345-0@lo:400:kworker.0 Request processed in 50us (1327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319269.318195:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 640 00000100:00100000:2.0:1767319269.318249:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982772480:0@lo:400:kworker.0 00000100:00100000:2.0:1767319269.318260:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982772352:0@lo:400:kworker.0 00000100:00100000:0.0:1767319269.318690:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982771840:0@lo:38:kworker.0 00000100:00100000:0.0:1767319269.318730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319269.318762:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982772096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319269.318797:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319269.318818:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982772224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319269.318852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319269.318873:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982772608:0@lo:38:kworker.0 00000100:00100000:0.0:1767319269.318894:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319269.318926:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982771840 00000100:00100000:0.0:1767319269.318944:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982771840 00000100:00100000:0.0:1767319269.318972:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772096 00000100:00100000:0.0:1767319269.318975:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772096 00000100:00100000:0.0:1767319269.318985:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772608 00000100:00100000:0.0:1767319269.318987:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772608 00000100:00100000:0.0:1767319269.318997:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1331 00000100:00100000:0.0:1767319269.319003:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982771840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.319021:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.319055:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982771840:12345-0@lo:38:kworker.0 Request processed in 51us (325us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.319061:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1331 00000100:00100000:0.0:1767319269.319074:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1332 00000100:00100000:0.0:1767319269.319077:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982772096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.319086:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.319107:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982772096:12345-0@lo:38:kworker.0 Request processed in 28us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.319111:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1332 00000100:00100000:0.0:1767319269.319121:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1333 00000100:00100000:0.0:1767319269.319125:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982772608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319269.319132:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319269.319151:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982772608:12345-0@lo:38:kworker.0 Request processed in 25us (257us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319269.319157:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1333 00000100:00080000:0.0:1767319269.319185:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319269.319190:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319269.319194:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982771840:0@lo:38:kworker.0 00000100:00080000:0.0:1767319269.319201:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319269.319204:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319269.319207:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982772096:0@lo:38:kworker.0 00000100:00080000:0.0:1767319269.319213:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319269.319216:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319269.319219:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982772608:0@lo:38:kworker.0 00000100:00100000:1.0:1767319269.319370:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772224 00000100:00100000:1.0:1767319269.319374:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772224 00000100:00100000:1.0:1767319269.319399:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 358 00000100:00100000:1.0:1767319269.319404:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982772224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319269.319414:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319269.319453:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982772224:12345-0@lo:38:kworker.0 Request processed in 48us (600us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319269.319459:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 358 00000100:00080000:0.0:1767319269.320135:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319269.320143:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319269.320146:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982772224:0@lo:38:kworker.0 00010000:00010000:1.0:1767319269.508518:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319269.828574:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319269.828599:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982772736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319269.828672:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319269.828781:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772736 00000100:00100000:2.0:1767319269.828790:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772736 00000100:00100000:2.0:1767319269.828855:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 692 00000100:00100000:2.0:1767319269.828865:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982772736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319269.828901:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319269.828957:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982772736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319269.828965:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 692 00000100:00100000:3.0:1767319269.829089:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982772864:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319269.829119:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319269.829219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982772736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319269.829338:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772864 00000100:00100000:2.0:1767319269.829343:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772864 00000100:00100000:2.0:1767319269.829422:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 693 00000100:00100000:2.0:1767319269.829430:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982772864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319269.829449:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319269.829497:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982772864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (376us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319269.829504:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 693 00000100:00100000:0.0:1767319269.829571:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982772864:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319270.532499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319271.316453:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319271.316588:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319271.316722:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615488 00000100:00100000:3.0:1767319271.316733:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615488 00000100:00100000:3.0:1767319271.316869:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615616 00000100:00100000:3.0:1767319271.316874:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615616 00000100:00080000:2.0:1767319271.316956:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319271.318102:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1015 00000100:00100000:1.0:1767319271.318122:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915615616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319271.318304:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915615616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 190us (1715us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319271.318314:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1015 00000100:00100000:0.0:1767319271.318336:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1334 00000100:00100000:0.0:1767319271.318344:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915615488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319271.318367:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319271.318405:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915615488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (1965us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319271.318411:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1334 00010000:00010000:1.0:1767319271.556631:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319272.581700:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319273.604674:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319274.437520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319274.437531:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319274.437613:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982772992:0@lo:400:kworker.0 00000100:00100000:2.0:1767319274.437698:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319274.437803:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982772992 00000100:00100000:3.0:1767319274.437811:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982772992 00000100:00100000:3.0:1767319274.437879:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016 00000100:00100000:3.0:1767319274.437888:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982772992:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319274.437961:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319274.438127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319274.438133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319274.438136:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319274.438142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4517 00000100:00080000:0.0:1767319274.438146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319274.438151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319274.438174:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320380 x1853163982773120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319274.438240:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982773248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319274.438300:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.438338:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773248 00000100:00100000:1.0:1767319274.438342:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773248 00000100:00100000:1.0:1767319274.438374:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017 00000100:00100000:1.0:1767319274.438381:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982773248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319274.438416:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982773248:12345-0@lo:400:kworker.0 Request processed in 35us (144us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319274.438423:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017 00000100:00080000:1.0:1767319274.438446:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319274.438469:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320000 x1853163982773248/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319290 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319274.438482:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982773248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319274.438516:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982773120:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.438539:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319274.438564:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773120 00000100:00100000:1.0:1767319274.438568:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773120 00000100:00100000:1.0:1767319274.438585:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1335 00000100:00100000:1.0:1767319274.438590:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982773120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319274.438607:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319274.438641:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982773120:12345-0@lo:38:kworker.0 Request processed in 49us (100us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319274.438646:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1335 00000100:00080000:1.0:1767319274.438681:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.438686:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.438689:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982773120:0@lo:38:kworker.0 00000100:00100000:3.0:1767319274.439490:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982772992:12345-0@lo:400:kworker.0 Request processed in 1599us (1787us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319274.439499:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016 00010000:00080000:2.0:1767319274.439604:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321f80 x1853163982772992/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319290 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319274.439616:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982772992:0@lo:400:kworker.0 00000100:00080000:0.0:1767319274.440416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319274.440424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319274.440427:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319274.440433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4517 00000100:00080000:0.0:1767319274.440437:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319274.440443:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319274.440478:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982773376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319274.440551:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982773376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.440603:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319274.440692:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773376 00000100:00100000:3.0:1767319274.440699:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773376 00000100:00100000:3.0:1767319274.440762:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1336 00000100:00100000:3.0:1767319274.440770:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982773376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319274.440788:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319274.440834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319274.440839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319274.440841:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319274.440844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4517 00000100:00080000:0.0:1767319274.440846:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319274.440850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319274.440866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321c00 x1853163982773504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319274.445816:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982773504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.445902:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.446038:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773504 00000100:00100000:2.0:1767319274.446048:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773504 00000100:00100000:2.0:1767319274.446097:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 359 00000100:00100000:2.0:1767319274.446105:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982773504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319274.446128:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319274.446172:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982773504:12345-0@lo:38:kworker.0 Request processed in 67us (270us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319274.446180:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 359 00000100:00100000:3.0:1767319274.446357:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982773376:12345-0@lo:38:kworker.0 Request processed in 5586us (5752us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319274.446365:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1336 00000100:00080000:0.0:1767319274.446731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319274.446745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319274.446748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319274.446754:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4517 00000100:00080000:0.0:1767319274.446758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319274.446764:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319274.446782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320a80 x1853163982773888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319274.446930:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982773760:0@lo:400:kworker.0 00000100:00100000:3.0:1767319274.446965:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319274.446999:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319274.447006:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982773632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319274.447028:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.447097:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773760 00000100:00100000:2.0:1767319274.447102:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773760 00000100:00100000:2.0:1767319274.447153:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773632 00000100:00100000:2.0:1767319274.447156:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773632 00000100:00100000:2.0:1767319274.447176:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 641 00000100:00100000:2.0:1767319274.447182:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982773760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319274.447229:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982773760:12345-0@lo:400:kworker.0 Request processed in 46us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.447236:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 641 00000100:00100000:2.0:1767319274.447261:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 642 00000100:00100000:2.0:1767319274.447267:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982773632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319274.447316:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982773632:12345-0@lo:400:kworker.0 Request processed in 49us (287us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.447322:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 642 00000100:00100000:2.0:1767319274.447366:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982773632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319274.447377:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982773760:0@lo:400:kworker.0 00000100:00080000:3.0:1767319274.447446:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319274.447450:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319274.447962:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.447968:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.447971:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982773376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.447989:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982773888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319274.448022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319274.448057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.448061:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.448065:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982773504:0@lo:38:kworker.0 00000100:00100000:3.0:1767319274.448132:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982773888 00000100:00100000:3.0:1767319274.448137:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982773888 00000100:00100000:3.0:1767319274.448169:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1337 00000100:00100000:3.0:1767319274.448175:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982773888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319274.448192:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319274.448234:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982773888:12345-0@lo:38:kworker.0 Request processed in 58us (211us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319274.448240:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1337 00000100:00080000:1.0:1767319274.448499:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319274.448504:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319274.448507:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982773888:0@lo:38:kworker.0 00010000:00010000:1.0:1767319274.628494:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319274.948667:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319274.948687:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982774016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319274.948768:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319274.949507:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774016 00000100:00100000:2.0:1767319274.949516:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774016 00000100:00100000:2.0:1767319274.949574:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 694 00000100:00100000:2.0:1767319274.949583:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982774016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319274.949611:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319274.949669:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982774016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (901us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319274.949679:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 694 00000100:00100000:1.0:1767319274.949777:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982774016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319274.950263:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982774144:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319274.950318:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319274.950454:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774144 00000100:00100000:0.0:1767319274.950460:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774144 00000100:00100000:0.0:1767319274.950499:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 695 00000100:00100000:0.0:1767319274.950507:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982774144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319274.950526:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319274.950567:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982774144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319274.950574:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 695 00000100:00100000:3.0:1767319274.950704:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982774144:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319275.652465:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319276.427557:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319276.427691:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615744 00000100:00100000:2.0:1767319276.427718:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615744 00000100:00100000:2.0:1767319276.427771:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1338 00000100:00100000:2.0:1767319276.427779:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915615744:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319276.427805:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319276.427849:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319276.427874:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915615744:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 94us (325us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319276.427883:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1338 00000100:00100000:2.0:1767319276.427899:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915615872 00000100:00100000:2.0:1767319276.427903:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915615872 00000100:00080000:3.0:1767319276.427944:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319276.427949:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1018 00000100:00100000:2.0:1767319276.427958:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615872:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319276.428010:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915615872:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 53us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319276.428016:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1018 00010000:00010000:1.0:1767319276.676638:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319277.701733:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319278.725029:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319279.556394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319279.556406:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319279.556475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319279.556479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319279.556483:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319279.556491:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4522 00000100:00080000:0.0:1767319279.556495:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319279.556500:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319279.556523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982774400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319279.556555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319279.556559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319279.556561:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319279.556565:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4522 00000100:00080000:0.0:1767319279.556569:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319279.556575:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319279.556580:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982774528:0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.556591:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322680 x1853163982774656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319279.556616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319279.556619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319279.556621:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319279.556625:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4522 00000100:00080000:0.0:1767319279.556628:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319279.556632:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319279.556658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982774784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319279.556700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319279.556703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319279.556705:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319279.556709:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4522 00000100:00080000:0.0:1767319279.556711:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319279.556714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319279.556728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323b80 x1853163982775168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319279.556773:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982774912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.556864:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982774272:0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.558298:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319279.558388:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319279.558400:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982775040:0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.558438:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319279.558485:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774912 00000100:00100000:0.0:1767319279.558494:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774912 00000100:00100000:0.0:1767319279.558555:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775040 00000100:00100000:0.0:1767319279.558558:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775040 00000100:00100000:0.0:1767319279.558583:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 643 00000100:00100000:0.0:1767319279.558592:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982774912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.558662:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982774912:12345-0@lo:400:kworker.0 Request processed in 70us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319279.558671:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 643 00000100:00100000:0.0:1767319279.558688:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 644 00000100:00100000:0.0:1767319279.558696:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982775040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.558725:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982775040:12345-0@lo:400:kworker.0 Request processed in 30us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319279.558731:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 644 00000100:00100000:0.0:1767319279.558781:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982775040:0@lo:400:kworker.0 00000100:00100000:0.0:1767319279.558792:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982774912:0@lo:400:kworker.0 00000100:00080000:0.0:1767319279.558837:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319279.558842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319279.559127:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982774400:0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.559170:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319279.559211:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982774656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.559234:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319279.559257:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982774784:0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.559301:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319279.559305:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774400 00000100:00100000:0.0:1767319279.559312:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774400 00000100:00100000:1.0:1767319279.559329:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982775168:0@lo:38:kworker.0 00000100:00100000:0.0:1767319279.559339:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774656 00000100:00100000:0.0:1767319279.559344:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774656 00000100:00100000:1.0:1767319279.559353:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319279.559368:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775168 00000100:00100000:0.0:1767319279.559372:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775168 00000100:00100000:0.0:1767319279.559390:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1339 00000100:00100000:1.0:1767319279.559399:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1340 00000100:00100000:0.0:1767319279.559400:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982774400:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319279.559410:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982774656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319279.559421:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767319279.559425:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319279.559514:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982774656:12345-0@lo:38:kworker.0 Request processed in 104us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319279.559521:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1340 00000100:00100000:1.0:1767319279.559537:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1341 00000100:00100000:1.0:1767319279.559541:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982775168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319279.559551:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319279.559588:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982775168:12345-0@lo:38:kworker.0 Request processed in 45us (235us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319279.559593:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1341 00000100:00100000:1.0:1767319279.559613:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774784 00000100:00100000:1.0:1767319279.559617:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774784 00000100:00100000:1.0:1767319279.559639:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 360 00000100:00100000:1.0:1767319279.559656:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982774784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319279.559666:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319279.559686:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982774400:12345-0@lo:38:kworker.0 Request processed in 288us (514us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319279.559694:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1339 00000100:00100000:1.0:1767319279.559704:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982774784:12345-0@lo:38:kworker.0 Request processed in 46us (402us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319279.559712:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 360 00000100:00080000:0.0:1767319279.559751:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319279.559758:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319279.559763:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982774400:0@lo:38:kworker.0 00000100:00080000:0.0:1767319279.559773:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319279.559777:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319279.559780:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982774656:0@lo:38:kworker.0 00000100:00080000:0.0:1767319279.559789:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319279.559791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319279.559795:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982774784:0@lo:38:kworker.0 00000100:00080000:0.0:1767319279.559803:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319279.559806:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319279.559808:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982775168:0@lo:38:kworker.0 00000100:00100000:2.0:1767319279.561752:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319279.562680:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319279.563188:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774272 00000100:00100000:2.0:1767319279.563203:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774272 00000100:00100000:2.0:1767319279.563311:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982774528 00000100:00100000:2.0:1767319279.563314:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982774528 00000100:00100000:2.0:1767319279.563341:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019 00000100:00100000:2.0:1767319279.563351:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982774272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.563443:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982774272:12345-0@lo:400:kworker.0 Request processed in 93us (1698us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319279.563452:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019 00000100:00100000:2.0:1767319279.563471:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020 00000100:00100000:2.0:1767319279.563477:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982774528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319279.563508:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982774528:12345-0@lo:400:kworker.0 Request processed in 31us (836us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319279.563514:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020 00010000:00080000:2.0:1767319279.563564:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321c00 x1853163982774272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319295 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319279.563788:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982774272:0@lo:400:kworker.0 00010000:00080000:3.0:1767319279.564129:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320e00 x1853163982774528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319295 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319279.564140:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982774528:0@lo:400:kworker.0 00000100:00080000:0.0:1767319279.567120:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319279.748530:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319280.069368:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319280.069391:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982775296:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319280.069461:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319280.070425:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775296 00000100:00100000:2.0:1767319280.070442:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775296 00000100:00100000:2.0:1767319280.070547:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00100000:0.0:1767319280.070548:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319280.070563:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982775296:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319280.070579:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982775424:0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319280.070614:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319280.070663:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319280.070719:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982775296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 159us (1254us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319280.070729:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00100000:2.0:1767319280.070752:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775424 00000100:00100000:2.0:1767319280.070756:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775424 00000100:00100000:2.0:1767319280.070788:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 697 00000100:00100000:2.0:1767319280.070795:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982775424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319280.070816:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319280.070856:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982775424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319280.070862:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 697 00000100:00100000:0.0:1767319280.071088:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982775424:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319280.071703:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982775296:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319280.772489:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319281.552403:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319281.552474:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319281.552658:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616128 00000100:00100000:0.0:1767319281.552673:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616128 00000100:00100000:0.0:1767319281.552777:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616000 00000100:00100000:0.0:1767319281.552781:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616000 00000100:00100000:0.0:1767319281.552817:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1021 00000100:00100000:0.0:1767319281.552831:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915616128:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319281.552937:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915616128:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 110us (551us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319281.552946:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1021 00000100:00100000:0.0:1767319281.552966:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1342 00000100:00100000:0.0:1767319281.552972:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915616000:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319281.552996:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319281.553024:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915616000:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (550us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319281.553031:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1342 00000100:00080000:3.0:1767319281.553505:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319281.796588:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319282.823763:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319283.844684:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319284.679173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.679187:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319284.679298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.679303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.679308:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.679317:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.679323:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.679331:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319284.679359:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988a80 x1853163982775680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319284.679387:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:3.0:1767319284.679403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767319284.679404:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982775552:0@lo:400:kworker.0 00000100:00080000:3.0:1767319284.679406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.679408:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.679413:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.679415:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.679419:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319284.679429:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988700 x1853163982775936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319284.679436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.679439:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.679440:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.679442:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.679445:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.679448:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319284.679463:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966a00 x1853163982776064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319284.679489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319284.679499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319284.679503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319284.679504:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319284.679507:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4528 00000100:00080000:3.0:1767319284.679509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319284.679512:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319284.679523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2300 x1853163982776448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319284.679557:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982776192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.679588:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982775808:0@lo:400:kworker.0 00000100:00100000:3.0:1767319284.679592:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.679663:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982775680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319284.680124:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776192 00000100:00100000:0.0:1767319284.680137:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776192 00000100:00100000:0.0:1767319284.680213:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 645 00000100:00100000:0.0:1767319284.680224:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982776192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.686118:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.686254:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775552 00000100:00100000:1.0:1767319284.686267:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775552 00000100:00100000:1.0:1767319284.686685:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775808 00000100:00100000:1.0:1767319284.686691:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775808 00000100:00100000:1.0:1767319284.686734:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1022 00000100:00100000:1.0:1767319284.686745:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982775552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.686835:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982775552:12345-0@lo:400:kworker.0 Request processed in 92us (7345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319284.686843:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1022 00000100:00100000:1.0:1767319284.686864:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1767319284.686871:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982775808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.686903:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982775808:12345-0@lo:400:kworker.0 Request processed in 33us (799us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319284.686910:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023 00010000:00080000:1.0:1767319284.686956:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631989180 x1853163982775808/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319300 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319284.686975:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982775808:0@lo:400:kworker.0 00010000:00080000:1.0:1767319284.686989:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0e00 x1853163982775552/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319300 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319284.687000:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982775552:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.687040:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982776320:0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.687340:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319284.687503:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776320 00000100:00100000:1.0:1767319284.687509:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776320 00000100:00100000:1.0:1767319284.687640:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 646 00000100:00100000:1.0:1767319284.687648:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982776320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319284.687689:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982776320:12345-0@lo:400:kworker.0 Request processed in 43us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319284.687696:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 646 00000100:00100000:1.0:1767319284.687916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982776320:0@lo:400:kworker.0 00000100:00080000:1.0:1767319284.687996:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319284.688001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319284.688005:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319284.695341:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.695417:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982775936:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.695470:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.695502:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982776064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.695531:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319284.695569:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776064 00000100:00100000:2.0:1767319284.695579:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776064 00000100:00100000:1.0:1767319284.695608:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775680 00000100:00100000:1.0:1767319284.695613:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775680 00000100:00100000:1.0:1767319284.695642:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982775936 00000100:00100000:1.0:1767319284.695645:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982775936 00000100:00100000:1.0:1767319284.695668:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1343 00000100:00100000:1.0:1767319284.695677:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982775680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319284.695714:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319284.695770:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982775680:12345-0@lo:38:kworker.0 Request processed in 93us (442us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319284.695778:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1343 00000100:00100000:1.0:1767319284.695796:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1344 00000100:00100000:1.0:1767319284.695799:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982775936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319284.695808:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319284.695831:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982775936:12345-0@lo:38:kworker.0 Request processed in 30us (360us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319284.695836:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 361 00000100:00100000:1.0:1767319284.695837:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1344 00000100:00100000:2.0:1767319284.695841:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982776064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319284.695851:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319284.695877:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982776064:12345-0@lo:38:kworker.0 Request processed in 35us (345us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319284.695884:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 361 00000100:00080000:2.0:1767319284.695928:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.695933:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.695937:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982776064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.695946:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982776448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.695974:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319284.696021:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.696025:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.696028:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982775680:0@lo:38:kworker.0 00000100:00080000:2.0:1767319284.696036:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.696040:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.696042:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982775936:0@lo:38:kworker.0 00000100:00100000:0.0:1767319284.696268:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982776192:12345-0@lo:400:kworker.0 Request processed in 16029us (16657us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319284.696295:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 645 00000100:00100000:3.0:1767319284.696584:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776448 00000100:00100000:3.0:1767319284.696590:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776448 00000100:00100000:3.0:1767319284.696788:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1345 00000100:00100000:3.0:1767319284.696795:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982776448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319284.696808:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319284.696859:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982776448:12345-0@lo:38:kworker.0 Request processed in 63us (883us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319284.696865:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1345 00000100:00080000:2.0:1767319284.697775:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319284.697785:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319284.697790:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982776448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319284.697840:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982776192:0@lo:400:kworker.0 00010000:00010000:1.0:1767319284.868601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319285.188600:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982776576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319285.188695:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319285.189514:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776576 00000100:00100000:2.0:1767319285.189527:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776576 00000100:00100000:2.0:1767319285.189607:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00100000:2.0:1767319285.189619:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982776576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319285.189660:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319285.189735:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982776576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (1039us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319285.189743:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767319285.190198:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982776576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319285.190224:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319285.190232:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982776704:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319285.190302:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319285.194442:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776704 00000100:00100000:2.0:1767319285.194464:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776704 00000100:00100000:2.0:1767319285.194571:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000100:00100000:2.0:1767319285.194584:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982776704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319285.194644:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319285.194774:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982776704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 192us (4470us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319285.194783:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000100:00100000:1.0:1767319285.194958:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982776704:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319285.893562:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319286.670915:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319286.671037:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616512 00000100:00100000:0.0:1767319286.671055:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616512 00000100:00100000:0.0:1767319286.671183:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1024 00000100:00100000:0.0:1767319286.671197:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915616512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319286.671374:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915616512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 179us (469us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319286.671383:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1024 00000100:00080000:1.0:1767319286.671626:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767319286.672667:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915616256 offset 0 length 520: 4 00000100:00100000:3.0:1767319286.672779:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319286.673115:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616384 00000100:00100000:2.0:1767319286.673126:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616384 00000100:00100000:2.0:1767319286.673183:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1346 00000100:00100000:2.0:1767319286.673192:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915616384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319286.673214:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319286.677044:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915616384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3837us (4250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319286.677063:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1346 00010000:00010000:1.0:1767319286.917854:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319287.945595:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319288.964780:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319289.798367:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319289.798393:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319289.798481:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319289.798486:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319289.798491:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319289.798502:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4533 00000100:00080000:2.0:1767319289.798508:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319289.798519:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319289.798553:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982776960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319289.798585:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319289.798589:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982776832:0@lo:400:kworker.0 00000100:00080000:2.0:1767319289.798591:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319289.798598:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319289.798611:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4533 00000100:00080000:2.0:1767319289.798615:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319289.798619:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319289.798634:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4000 x1853163982777216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319289.798644:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319289.798648:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319289.798650:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319289.798655:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4533 00000100:00080000:2.0:1767319289.798658:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319289.798663:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319289.798676:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4a80 x1853163982777344/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319289.798713:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319289.798723:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319289.798728:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319289.798730:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319289.798735:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4533 00000100:00080000:2.0:1767319289.798737:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319289.798741:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319289.798757:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017800 x1853163982777728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319289.798806:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776832 00000100:00100000:2.0:1767319289.798814:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776832 00000100:00100000:3.0:1767319289.798821:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319289.798833:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982777088:0@lo:400:kworker.0 00000100:00100000:3.0:1767319289.798872:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319289.798992:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982777472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319289.799031:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319264/real 1767319264] req@ffff93460a580e00 x1853163982770304/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319289 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319289.799053:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319289.799095:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319289.799104:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982777600:0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.799126:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319289.799154:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319289.799157:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777088 00000100:00100000:2.0:1767319289.799164:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777088 00000100:00100000:1.0:1767319289.799184:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025 00000100:00100000:1.0:1767319289.799206:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982776832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319289.799215:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026 00000100:00100000:2.0:1767319289.799229:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982777088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.799342:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982776832:12345-0@lo:400:kworker.0 Request processed in 145us (631us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319289.799352:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025 00000100:00100000:1.0:1767319289.799391:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777472 00000100:00100000:1.0:1767319289.799399:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777472 00000100:00100000:1.0:1767319289.799443:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777600 00000100:00100000:1.0:1767319289.799446:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777600 00000100:00100000:1.0:1767319289.799478:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 647 00000100:00100000:1.0:1767319289.799486:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982777472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.799525:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982777472:12345-0@lo:400:kworker.0 Request processed in 40us (488us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319289.799531:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 647 00000100:00100000:1.0:1767319289.799544:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 648 00000100:00100000:1.0:1767319289.799550:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982777600:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.799575:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982777600:12345-0@lo:400:kworker.0 Request processed in 25us (448us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319289.799581:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 648 00000100:00080000:1.0:1767319289.799607:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319289.799610:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319289.799612:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319289.799643:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982777600:0@lo:400:kworker.0 00000100:00100000:1.0:1767319289.799654:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982777472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319289.799727:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a580e00 x1853163982770304/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319289 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319289.799750:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982776960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319289.799795:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.799823:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982776960 00000100:00100000:3.0:1767319289.799830:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982776960 00000100:00100000:0.0:1767319289.799845:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982777216:0@lo:38:kworker.0 00000100:00100000:3.0:1767319289.799857:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347 00000100:00100000:3.0:1767319289.799866:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982776960:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319289.799880:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767319289.799909:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319289.799912:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982777344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319289.799955:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.799989:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982776960:12345-0@lo:38:kworker.0 Request processed in 119us (190us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319289.799996:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982777728:0@lo:38:kworker.0 00000100:00100000:3.0:1767319289.800004:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347 00000100:00100000:3.0:1767319289.800032:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777216 00000100:00100000:3.0:1767319289.800038:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777216 00000100:00100000:0.0:1767319289.800045:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319289.800066:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777728 00000100:00100000:3.0:1767319289.800072:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777728 00000100:00100000:3.0:1767319289.800083:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1348 00000100:00100000:3.0:1767319289.800088:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982777216:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319289.800097:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319289.800129:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982777216:12345-0@lo:38:kworker.0 Request processed in 41us (259us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319289.800135:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1348 00000100:00100000:3.0:1767319289.800145:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1349 00000100:00100000:3.0:1767319289.800148:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982777728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319289.800157:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319289.800179:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982777728:12345-0@lo:38:kworker.0 Request processed in 29us (140us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319289.800185:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1349 00000100:00100000:3.0:1767319289.800223:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777344 00000100:00100000:3.0:1767319289.800227:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777344 00000100:00100000:3.0:1767319289.800247:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 362 00000100:00100000:3.0:1767319289.800252:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982777344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319289.800261:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319289.800306:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982777344:12345-0@lo:38:kworker.0 Request processed in 53us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319289.800311:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 362 00000100:00100000:2.0:1767319289.803818:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982777088:12345-0@lo:400:kworker.0 Request processed in 4576us (4928us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319289.803839:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026 00000100:00080000:0.0:1767319289.804740:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319289.804766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319289.804774:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982770304:0@lo:250:kworker.0 00000100:00080000:0.0:1767319289.804853:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319289.804858:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319289.804862:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982776960:0@lo:38:kworker.0 00000100:00080000:0.0:1767319289.804876:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319289.804879:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319289.804882:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982777216:0@lo:38:kworker.0 00000100:00080000:0.0:1767319289.804895:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319289.804899:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319289.804902:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982777344:0@lo:38:kworker.0 00000100:00080000:0.0:1767319289.804914:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319289.804918:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319289.804921:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982777728:0@lo:38:kworker.0 00000100:00080000:2.0:1767319289.805029:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319289.805036:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319289.805040:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319289.805046:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4512 00000100:00080000:2.0:1767319289.805052:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319289.805059:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319289.805194:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163982777856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319289.805314:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982777856:0@lo:250:kworker.0 00000400:00000100:0.0:1767319289.805373:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982777856 offset 0 length 520: 4 00010000:00080000:1.0:1767319289.807495:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4380 x1853163982777088/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319305 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319289.807521:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982777088:0@lo:400:kworker.0 00010000:00080000:1.0:1767319289.807543:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f966a00 x1853163982776832/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319305 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319289.807552:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982776832:0@lo:400:kworker.0 00010000:00010000:1.0:1767319289.988865:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319290.308999:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319290.309031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982777984:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319290.309324:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319290.309396:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982777984 00000100:00100000:0.0:1767319290.309407:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982777984 00000100:00100000:0.0:1767319290.309486:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 700 00000100:00100000:0.0:1767319290.309498:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982777984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319290.309538:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319290.309556:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319290.309566:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982778112:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319290.309592:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982777984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (267us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319290.309595:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319290.309601:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 700 00000100:00100000:0.0:1767319290.309618:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778112 00000100:00100000:0.0:1767319290.309622:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778112 00000100:00100000:3.0:1767319290.310767:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982777984:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319290.316701:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 701 00000100:00100000:0.0:1767319290.316738:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982778112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319290.316786:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319290.316908:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982778112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 190us (7311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319290.316916:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 701 00000100:00100000:3.0:1767319290.317142:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982778112:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319291.012690:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319291.789736:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319291.789955:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616768 00000100:00100000:0.0:1767319291.789973:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616768 00000100:00100000:0.0:1767319291.790081:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1027 00000100:00100000:0.0:1767319291.790095:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915616768:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319291.790211:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915616768:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 119us (489us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319291.790221:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1027 00000100:00080000:1.0:1767319291.790343:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319291.790754:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319291.790984:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616640 00000100:00100000:2.0:1767319291.790992:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616640 00000100:00100000:2.0:1767319291.791037:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1350 00000100:00100000:2.0:1767319291.791044:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915616640:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319291.791070:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319291.791136:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915616640:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (383us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319291.791144:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1350 00010000:00010000:1.0:1767319292.037816:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319293.060797:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319294.085167:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319294.916621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319294.916635:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319294.916726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319294.916731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319294.916736:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319294.916747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4538 00000100:00080000:0.0:1767319294.916753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319294.916761:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319294.916791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982778368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319294.916827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319294.916831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319294.916833:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319294.916837:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4538 00000100:00080000:0.0:1767319294.916839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319294.916844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319294.916855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163982778624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319294.916864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319294.916866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319294.916868:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319294.916871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4538 00000100:00080000:0.0:1767319294.916873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319294.916877:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319294.916890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982778752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319294.916948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319294.916952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319294.916954:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319294.916957:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4538 00000100:00080000:0.0:1767319294.916960:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319294.916963:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319294.916981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163982779136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319294.917012:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982778880:0@lo:400:kworker.0 00000100:00100000:0.0:1767319294.917093:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319294.917147:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319294.917164:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982779008:0@lo:400:kworker.0 00000100:00100000:0.0:1767319294.917188:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.917322:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982778496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.917373:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319294.917399:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319294.917404:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982778240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.917427:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319294.917504:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778496 00000100:00100000:0.0:1767319294.917514:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778496 00000100:00100000:0.0:1767319294.917584:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778240 00000100:00100000:0.0:1767319294.917588:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778240 00000100:00100000:0.0:1767319294.917614:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767319294.917624:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982778496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319294.917682:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982778496:12345-0@lo:400:kworker.0 Request processed in 59us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319294.917692:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767319294.917710:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029 00000100:00100000:0.0:1767319294.917716:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982778240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319294.917742:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982778240:12345-0@lo:400:kworker.0 Request processed in 26us (315us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319294.917749:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029 00000100:00100000:2.0:1767319294.917824:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982778368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319294.917860:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319294.917887:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982778624:0@lo:38:kworker.0 00000100:00100000:2.0:1767319294.917909:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319294.917926:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982778752:0@lo:38:kworker.0 00000100:00100000:2.0:1767319294.917955:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319294.917975:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982779136:0@lo:38:kworker.0 00000100:00100000:2.0:1767319294.917999:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319294.918030:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778752 00000100:00100000:2.0:1767319294.918034:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778752 00000100:00100000:2.0:1767319294.918056:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 363 00000100:00100000:2.0:1767319294.918061:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982778752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319294.918084:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319294.918118:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982778752:12345-0@lo:38:kworker.0 Request processed in 56us (163us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319294.918125:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 363 00000100:00080000:2.0:1767319294.918163:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319294.918169:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319294.918173:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982778752:0@lo:38:kworker.0 00000100:00100000:2.0:1767319294.918198:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778368 00000100:00100000:2.0:1767319294.918201:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778368 00000100:00100000:2.0:1767319294.918212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778624 00000100:00100000:2.0:1767319294.918215:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778624 00000100:00100000:2.0:1767319294.918223:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779136 00000100:00100000:2.0:1767319294.918225:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779136 00000100:00100000:2.0:1767319294.918234:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1351 00000100:00100000:2.0:1767319294.918238:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982778368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319294.918249:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319294.918298:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982778368:12345-0@lo:38:kworker.0 Request processed in 57us (437us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319294.918305:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1351 00000100:00100000:2.0:1767319294.918316:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352 00000100:00100000:2.0:1767319294.918320:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982778624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319294.918329:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319294.918351:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982778624:12345-0@lo:38:kworker.0 Request processed in 29us (441us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319294.918378:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352 00000100:00100000:2.0:1767319294.918395:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1353 00000100:00100000:2.0:1767319294.918399:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982779136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319294.918408:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319294.918430:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982779136:12345-0@lo:38:kworker.0 Request processed in 30us (430us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319294.918436:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1353 00000100:00080000:2.0:1767319294.918457:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319294.918461:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319294.918464:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982778368:0@lo:38:kworker.0 00000100:00080000:2.0:1767319294.918472:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319294.918476:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319294.918478:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982778624:0@lo:38:kworker.0 00000100:00080000:2.0:1767319294.918485:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319294.918488:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319294.918491:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982779136:0@lo:38:kworker.0 00000100:00080000:3.0:1767319294.918821:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319294.918850:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f966a00 x1853163982778240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319310 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319294.918864:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982778240:0@lo:400:kworker.0 00010000:00080000:3.0:1767319294.918876:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a580e00 x1853163982778496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319310 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319294.918892:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982778496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319294.921289:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982778880 00000100:00100000:1.0:1767319294.921320:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982778880 00000100:00100000:1.0:1767319294.921437:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779008 00000100:00100000:1.0:1767319294.921442:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779008 00000100:00100000:1.0:1767319294.921472:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 649 00000100:00100000:1.0:1767319294.921484:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982778880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319294.921507:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 650 00000100:00100000:0.0:1767319294.921520:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982779008:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319294.921622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319294.921628:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319294.925674:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982779008:12345-0@lo:400:kworker.0 Request processed in 4144us (8471us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319294.925693:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 650 00000100:00100000:1.0:1767319294.926754:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982778880:12345-0@lo:400:kworker.0 Request processed in 5267us (9655us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319294.926769:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 649 00000100:00100000:3.0:1767319294.930014:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982779008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319294.930038:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982778880:0@lo:400:kworker.0 00010000:00010000:1.0:1767319295.108855:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319295.428549:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982779264:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319295.428650:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319295.428753:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982779392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319295.428788:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319295.428859:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779264 00000100:00100000:1.0:1767319295.428868:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779264 00000100:00100000:1.0:1767319295.428923:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779392 00000100:00100000:1.0:1767319295.428927:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779392 00000100:00100000:1.0:1767319295.428945:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767319295.428954:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982779264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319295.428990:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319295.429077:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 703 00000100:00100000:3.0:1767319295.429087:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982779392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319295.429104:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319295.429146:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982779392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319295.429154:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 703 00000100:00100000:3.0:1767319295.429241:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982779392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319295.439433:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982779264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 10464us (10767us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319295.439451:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00000100:00100000:0.0:1767319295.439613:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982779264:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319296.132905:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319296.911068:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319296.911127:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319296.911318:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915616896 00000100:00100000:1.0:1767319296.911341:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915616896 00000100:00100000:1.0:1767319296.911395:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617024 00000100:00100000:1.0:1767319296.911399:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617024 00000100:00100000:1.0:1767319296.911452:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1030 00000100:00100000:1.0:1767319296.911463:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915617024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319296.911560:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915617024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319296.911569:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1030 00000100:00100000:1.0:1767319296.911589:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1354 00000100:00100000:1.0:1767319296.911596:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915616896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319296.911619:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319296.911646:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915616896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (595us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319296.911653:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1354 00000100:00080000:1.0:1767319296.911670:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319297.164929:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319298.180601:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319299.204725:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319300.036692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319300.036707:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319300.036818:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982779520:0@lo:400:kworker.0 00000100:00100000:0.0:1767319300.036913:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319300.037026:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779520 00000100:00100000:1.0:1767319300.037036:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779520 00000100:00100000:1.0:1767319300.037129:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031 00000100:00100000:1.0:1767319300.037141:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982779520:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319300.037248:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319300.037698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319300.037704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319300.037709:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319300.037715:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4543 00000100:00080000:3.0:1767319300.037720:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319300.037727:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319300.037756:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982779648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319300.037858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982779776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319300.037902:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319300.037942:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779776 00000100:00100000:2.0:1767319300.037947:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779776 00000100:00100000:2.0:1767319300.037986:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032 00000100:00100000:2.0:1767319300.037994:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982779776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319300.038036:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982779776:12345-0@lo:400:kworker.0 Request processed in 42us (133us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319300.038044:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032 00010000:00080000:2.0:1767319300.038092:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638595500 x1853163982779776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319316 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319300.038107:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982779776:0@lo:400:kworker.0 00000100:00080000:2.0:1767319300.038128:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319300.038144:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982779648:0@lo:38:kworker.0 00000100:00100000:2.0:1767319300.038170:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319300.038197:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779648 00000100:00100000:2.0:1767319300.038201:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779648 00000100:00100000:2.0:1767319300.038221:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1355 00000100:00100000:2.0:1767319300.038226:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982779648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319300.038249:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319300.038318:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982779648:12345-0@lo:38:kworker.0 Request processed in 90us (146us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319300.038324:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1355 00000100:00080000:2.0:1767319300.038349:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319300.038354:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319300.038357:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982779648:0@lo:38:kworker.0 00000100:00100000:1.0:1767319300.047558:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982779520:12345-0@lo:400:kworker.0 Request processed in 10403us (10627us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319300.047578:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031 00010000:00080000:0.0:1767319300.048151:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059ced80 x1853163982779520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319316 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319300.048173:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982779520:0@lo:400:kworker.0 00000100:00080000:3.0:1767319300.048634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319300.048640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319300.048645:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319300.048651:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4543 00000100:00080000:3.0:1767319300.048656:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319300.048664:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319300.048719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163982779904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319300.048804:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982779904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319300.048886:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319300.049090:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982779904 00000100:00100000:1.0:1767319300.049099:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982779904 00000100:00100000:1.0:1767319300.049152:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1356 00000100:00100000:1.0:1767319300.049162:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982779904:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319300.049189:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319300.049365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319300.049369:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319300.049371:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319300.049376:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4543 00000100:00080000:3.0:1767319300.049379:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319300.049383:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319300.049404:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163982780032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319300.053129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982780032:0@lo:38:kworker.0 00000100:00100000:2.0:1767319300.053244:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319300.053435:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780032 00000100:00100000:0.0:1767319300.053447:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780032 00000100:00100000:0.0:1767319300.053497:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 364 00000100:00100000:0.0:1767319300.053506:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982780032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319300.053537:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319300.053591:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982780032:12345-0@lo:38:kworker.0 Request processed in 84us (345us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319300.053599:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 364 00000100:00100000:1.0:1767319300.054803:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982779904:12345-0@lo:38:kworker.0 Request processed in 5634us (5909us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319300.054818:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1356 00000100:00080000:2.0:1767319300.056402:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319300.056414:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319300.056420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982779904:0@lo:38:kworker.0 00000100:00080000:2.0:1767319300.056465:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319300.056470:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319300.056473:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982780032:0@lo:38:kworker.0 00000100:00100000:1.0:1767319300.057199:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982780160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319300.057262:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319300.057329:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780160 00000100:00100000:2.0:1767319300.057338:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780160 00000100:00100000:1.0:1767319300.057400:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319300.057407:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982780288:0@lo:400:kworker.0 00000100:00100000:2.0:1767319300.057418:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 651 00000100:00100000:2.0:1767319300.057429:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982780160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319300.057435:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319300.057468:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780288 00000100:00100000:1.0:1767319300.057472:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780288 00000100:00100000:2.0:1767319300.057494:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982780160:12345-0@lo:400:kworker.0 Request processed in 66us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319300.057502:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 651 00000100:00080000:2.0:1767319300.057525:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319300.057534:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982780160:0@lo:400:kworker.0 00000100:00080000:2.0:1767319300.057642:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319300.057649:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 652 00000100:00100000:0.0:1767319300.057660:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982780288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319300.057711:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982780288:12345-0@lo:400:kworker.0 Request processed in 53us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319300.057718:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 652 00000100:00100000:2.0:1767319300.057738:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982780288:0@lo:400:kworker.0 00000100:00080000:3.0:1767319300.058421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319300.058429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319300.058433:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319300.058438:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4543 00000100:00080000:3.0:1767319300.058444:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319300.058449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319300.058483:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595880 x1853163982780416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319300.059331:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982780416:0@lo:38:kworker.0 00000100:00100000:0.0:1767319300.059401:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319300.059585:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780416 00000100:00100000:1.0:1767319300.059593:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780416 00000100:00100000:1.0:1767319300.059636:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1357 00000100:00100000:1.0:1767319300.059644:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982780416:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319300.059670:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319300.059729:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982780416:12345-0@lo:38:kworker.0 Request processed in 85us (328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319300.059737:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1357 00000100:00080000:2.0:1767319300.059778:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319300.059785:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319300.059789:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982780416:0@lo:38:kworker.0 00010000:00010000:1.0:1767319300.228747:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319300.548563:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982780544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319300.548657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982780672:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319300.548959:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319300.549094:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780544 00000100:00100000:0.0:1767319300.549104:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780544 00000100:00100000:0.0:1767319300.549182:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00100000:0.0:1767319300.549193:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982780544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319300.549235:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319300.549585:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319300.549676:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780672 00000100:00100000:2.0:1767319300.549683:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780672 00000100:00100000:0.0:1767319300.549714:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982780544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 521us (755us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319300.549725:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00000100:00100000:2.0:1767319300.549755:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 705 00000100:00100000:2.0:1767319300.549766:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982780672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319300.549787:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319300.549830:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982780672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319300.549921:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982780544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319300.550022:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982780672:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319300.550063:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 705 00010000:00010000:1.0:1767319301.252642:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319302.028341:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319302.028675:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617280 00000100:00100000:1.0:1767319302.028701:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617280 00000100:00100000:1.0:1767319302.028787:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1033 00000100:00100000:1.0:1767319302.028798:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915617280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319302.028885:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915617280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (556us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319302.028892:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1033 00000100:00080000:2.0:1767319302.029295:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319302.033221:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319302.033380:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617152 00000100:00100000:1.0:1767319302.033391:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617152 00000100:00100000:1.0:1767319302.033442:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1358 00000100:00100000:1.0:1767319302.033451:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617152:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319302.033475:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319302.036335:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 2870us (3111us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319302.036362:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1358 00010000:00010000:1.0:1767319302.277291:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319303.300486:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319304.330039:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319305.156491:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319305.156583:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319305.156889:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982780800:0@lo:400:kworker.0 00000100:00100000:1.0:1767319305.156979:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319305.157098:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780800 00000100:00100000:0.0:1767319305.157107:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780800 00000100:00100000:0.0:1767319305.157215:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034 00000100:00100000:0.0:1767319305.157236:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982780800:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319305.157536:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319305.157542:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319305.157545:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319305.157551:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4548 00000100:00080000:2.0:1767319305.157555:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319305.157561:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319305.157584:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580e00 x1853163982780928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319305.157660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319305.157677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982781056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319305.157709:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319305.157747:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781056 00000100:00100000:3.0:1767319305.157751:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781056 00000100:00100000:3.0:1767319305.157786:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035 00000100:00100000:3.0:1767319305.157793:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982781056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319305.157828:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982781056:12345-0@lo:400:kworker.0 Request processed in 35us (118us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319305.157835:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035 00000100:00080000:3.0:1767319305.157873:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319305.157895:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f966a00 x1853163982781056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319321 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319305.157906:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982781056:0@lo:400:kworker.0 00000100:00100000:3.0:1767319305.157943:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982780928:0@lo:38:kworker.0 00000100:00100000:3.0:1767319305.157967:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319305.157998:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982780928 00000100:00100000:3.0:1767319305.158002:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982780928 00000100:00100000:3.0:1767319305.158019:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1359 00000100:00100000:3.0:1767319305.158023:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982780928:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319305.158041:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319305.158072:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982780928:12345-0@lo:38:kworker.0 Request processed in 48us (104us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319305.158078:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1359 00000100:00080000:3.0:1767319305.158101:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319305.158105:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319305.158108:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982780928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319305.171597:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982780800:12345-0@lo:400:kworker.0 Request processed in 14364us (14607us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319305.171614:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034 00000100:00080000:2.0:1767319305.173825:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319305.173834:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319305.173837:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319305.173852:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4548 00000100:00080000:2.0:1767319305.173856:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319305.173863:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319305.173902:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982781184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319305.173927:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319305.173930:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319305.173932:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319305.173935:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4548 00000100:00080000:2.0:1767319305.173937:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319305.173940:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319305.173952:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966a00 x1853163982781312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319305.173990:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319305.173993:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319305.173995:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319305.173998:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4548 00000100:00080000:2.0:1767319305.174000:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319305.174002:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319305.174010:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163982781696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319305.174070:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982781440:0@lo:400:kworker.0 00000100:00100000:2.0:1767319305.174125:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319305.174184:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319305.174190:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982781568:0@lo:400:kworker.0 00000100:00100000:2.0:1767319305.174211:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319305.174247:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781440 00000100:00100000:2.0:1767319305.174253:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781440 00000100:00100000:2.0:1767319305.174322:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781568 00000100:00100000:2.0:1767319305.174326:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781568 00000100:00100000:2.0:1767319305.174350:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 653 00000100:00100000:2.0:1767319305.174357:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982781440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319305.174405:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982781440:12345-0@lo:400:kworker.0 Request processed in 48us (279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319305.174411:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 653 00000100:00100000:2.0:1767319305.174427:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 654 00000100:00100000:2.0:1767319305.174433:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982781568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319305.174458:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982781568:12345-0@lo:400:kworker.0 Request processed in 27us (246us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319305.174463:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 654 00000100:00080000:2.0:1767319305.174477:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319305.174481:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319305.174757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982781184:0@lo:38:kworker.0 00000100:00100000:3.0:1767319305.174789:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319305.174814:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982781312:0@lo:38:kworker.0 00000100:00100000:3.0:1767319305.174847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319305.174868:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982781696:0@lo:38:kworker.0 00000100:00100000:3.0:1767319305.174887:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319305.174919:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781184 00000100:00100000:3.0:1767319305.174925:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781184 00000100:00100000:3.0:1767319305.174948:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781696 00000100:00100000:3.0:1767319305.174950:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781696 00000100:00100000:3.0:1767319305.174965:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1360 00000100:00100000:3.0:1767319305.174970:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982781184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319305.174987:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319305.175019:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982781184:12345-0@lo:38:kworker.0 Request processed in 48us (228us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319305.175025:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1360 00000100:00100000:3.0:1767319305.175037:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1361 00000100:00100000:3.0:1767319305.175041:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982781696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319305.175051:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319305.175073:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982781696:12345-0@lo:38:kworker.0 Request processed in 31us (185us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319305.175079:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1361 00000100:00080000:3.0:1767319305.175109:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319305.175115:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319305.175119:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982781184:0@lo:38:kworker.0 00000100:00080000:3.0:1767319305.175127:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319305.175130:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319305.175132:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982781696:0@lo:38:kworker.0 00000100:00100000:2.0:1767319305.175436:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982781568:0@lo:400:kworker.0 00000100:00100000:0.0:1767319305.176662:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781312 00000100:00100000:0.0:1767319305.176672:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781312 00000100:00100000:0.0:1767319305.176719:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 365 00000100:00100000:0.0:1767319305.176726:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982781312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319305.176745:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319305.176812:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982781312:12345-0@lo:38:kworker.0 Request processed in 86us (1974us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319305.176819:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 365 00000100:00080000:0.0:1767319305.176871:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319305.176876:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319305.176880:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982781312:0@lo:38:kworker.0 00010000:00080000:1.0:1767319305.177396:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583100 x1853163982780800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319321 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319305.177409:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982780800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319305.177831:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982781440:0@lo:400:kworker.0 00010000:00010000:1.0:1767319305.348721:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319305.668791:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319305.668814:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982781824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319305.668882:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319305.669037:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781824 00000100:00100000:2.0:1767319305.669046:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781824 00000100:00100000:2.0:1767319305.669099:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 706 00000100:00100000:2.0:1767319305.669109:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982781824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319305.669138:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319305.669194:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982781824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (311us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319305.669201:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767319305.669555:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319305.669565:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982781952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319305.669594:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319305.669661:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982781952 00000100:00100000:0.0:1767319305.669665:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982781952 00000100:00100000:0.0:1767319305.669694:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00100000:0.0:1767319305.669700:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982781952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319305.669720:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319305.669756:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982781952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (161us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319305.669762:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00100000:1.0:1767319305.669856:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982781952:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319305.670217:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982781824:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319306.372728:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319307.151387:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319307.151568:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617408 00000100:00100000:0.0:1767319307.151580:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617408 00000100:00100000:0.0:1767319307.151639:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1362 00000100:00100000:0.0:1767319307.151648:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319307.151672:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319307.151744:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (367us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319307.151751:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1362 00000100:00100000:2.0:1767319307.156312:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319307.156418:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617536 00000100:00100000:3.0:1767319307.156429:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617536 00000100:00100000:3.0:1767319307.156547:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1036 00000100:00100000:3.0:1767319307.156558:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915617536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319307.156643:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915617536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319307.156651:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1036 00000100:00080000:2.0:1767319307.157939:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319307.398105:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319308.420675:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319309.445346:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319310.276565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319310.276580:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319310.276681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319310.276686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319310.276691:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319310.276701:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4553 00000100:00080000:2.0:1767319310.276708:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319310.276716:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319310.276743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163982782208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319310.276769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319310.276772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319310.276773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319310.276777:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4553 00000100:00080000:2.0:1767319310.276780:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319310.276784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319310.276793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37100 x1853163982782464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319310.276801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319310.276805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319310.276807:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319310.276810:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4553 00000100:00080000:2.0:1767319310.276813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319310.276816:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319310.276827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163982782592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319310.276861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319310.276865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319310.276867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319310.276870:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4553 00000100:00080000:2.0:1767319310.276873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319310.276877:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319310.276888:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163982782976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319310.276983:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982782336:0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277093:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319310.277181:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319310.277188:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982782080:0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277214:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319310.277252:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982782720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277350:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319310.277372:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319310.277378:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982782848:0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277401:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319310.277435:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782336 00000100:00100000:1.0:1767319310.277445:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782336 00000100:00100000:1.0:1767319310.277516:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782080 00000100:00100000:1.0:1767319310.277519:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782080 00000100:00100000:1.0:1767319310.277548:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037 00000100:00100000:1.0:1767319310.277559:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982782336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277617:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982782336:12345-0@lo:400:kworker.0 Request processed in 60us (524us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.277626:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037 00000100:00100000:1.0:1767319310.277646:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038 00000100:00100000:1.0:1767319310.277653:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982782080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277680:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982782080:12345-0@lo:400:kworker.0 Request processed in 28us (465us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.277686:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038 00000100:00100000:1.0:1767319310.277705:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782720 00000100:00100000:1.0:1767319310.277709:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782720 00000100:00100000:1.0:1767319310.277739:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782848 00000100:00100000:1.0:1767319310.277742:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782848 00000100:00100000:1.0:1767319310.277766:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767319310.277773:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982782720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277808:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982782720:12345-0@lo:400:kworker.0 Request processed in 35us (457us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.277814:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767319310.277825:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767319310.277831:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982782848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277858:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982782848:12345-0@lo:400:kworker.0 Request processed in 27us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.277862:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 656 00000100:00080000:1.0:1767319310.277878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319310.277883:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319310.277886:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319310.277933:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982782848:0@lo:400:kworker.0 00000100:00100000:1.0:1767319310.277946:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982782720:0@lo:400:kworker.0 00010000:00080000:1.0:1767319310.277998:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a37480 x1853163982782080/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319326 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319310.278008:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982782080:0@lo:400:kworker.0 00010000:00080000:1.0:1767319310.278019:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34a80 x1853163982782336/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319326 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319310.278029:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982782336:0@lo:400:kworker.0 00000100:00100000:0.0:1767319310.278486:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982782208:0@lo:38:kworker.0 00000100:00100000:0.0:1767319310.278536:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.278584:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982782464:0@lo:38:kworker.0 00000100:00100000:0.0:1767319310.278608:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.278637:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982782592:0@lo:38:kworker.0 00000100:00100000:0.0:1767319310.278667:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.278693:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982782976:0@lo:38:kworker.0 00000100:00100000:0.0:1767319310.278715:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.278750:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782592 00000100:00100000:0.0:1767319310.278756:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782592 00000100:00100000:0.0:1767319310.278784:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 366 00000100:00100000:0.0:1767319310.278791:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982782592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319310.278814:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319310.278856:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982782592:12345-0@lo:38:kworker.0 Request processed in 64us (188us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319310.278863:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 366 00000100:00100000:0.0:1767319310.278887:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782208 00000100:00100000:0.0:1767319310.278891:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782208 00000100:00100000:0.0:1767319310.278906:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782464 00000100:00100000:0.0:1767319310.278908:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782464 00000100:00100000:0.0:1767319310.278915:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982782976 00000100:00100000:0.0:1767319310.278917:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982782976 00000100:00100000:0.0:1767319310.278925:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1363 00000100:00100000:0.0:1767319310.278929:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782208:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319310.278938:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319310.278962:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782208:12345-0@lo:38:kworker.0 Request processed in 31us (426us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319310.278968:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1363 00000100:00100000:0.0:1767319310.278977:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1364 00000100:00100000:0.0:1767319310.278981:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319310.278989:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319310.279007:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782464:12345-0@lo:38:kworker.0 Request processed in 24us (398us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319310.279012:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1364 00000100:00100000:0.0:1767319310.279021:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1365 00000100:00100000:0.0:1767319310.279025:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319310.279032:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319310.279050:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982782976:12345-0@lo:38:kworker.0 Request processed in 24us (334us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319310.279055:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1365 00000100:00080000:0.0:1767319310.279093:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319310.279098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319310.279679:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982782208:0@lo:38:kworker.0 00000100:00080000:0.0:1767319310.279701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319310.279705:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319310.279708:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982782464:0@lo:38:kworker.0 00000100:00080000:0.0:1767319310.279718:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319310.279721:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319310.279724:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982782592:0@lo:38:kworker.0 00000100:00080000:0.0:1767319310.279732:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319310.279736:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319310.279738:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982782976:0@lo:38:kworker.0 00010000:00010000:1.0:1767319310.470067:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319310.788616:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319310.788646:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982783104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319310.788732:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319310.788856:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783104 00000100:00100000:0.0:1767319310.788867:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783104 00000100:00100000:0.0:1767319310.788927:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00100000:0.0:1767319310.788937:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982783104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319310.788982:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319310.789036:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982783104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319310.789043:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00100000:3.0:1767319310.789097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982783104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319310.789586:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319310.789597:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982783232:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319310.789634:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319310.789708:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783232 00000100:00100000:1.0:1767319310.789713:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783232 00000100:00100000:1.0:1767319310.789747:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 709 00000100:00100000:1.0:1767319310.789754:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982783232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319310.789775:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319310.789817:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982783232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (183us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319310.789824:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 709 00000100:00100000:2.0:1767319310.789916:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982783232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319311.247262:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319311.247346:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617920 00000100:00100000:0.0:1767319311.247358:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617920 00000400:00000100:1.0:1767319311.247367:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915617664 offset 0 length 520: 4 00000100:00100000:1.0:1767319311.247390:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319311.247839:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915617792 00000100:00100000:0.0:1767319311.247844:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915617792 00000100:00100000:0.0:1767319311.247864:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1039 00000100:00100000:0.0:1767319311.247874:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915617920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319311.247957:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915617920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (704us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319311.247965:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1039 00000100:00100000:0.0:1767319311.247985:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1366 00000100:00100000:0.0:1767319311.247990:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617792:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:1.0:1767319311.247995:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:0.0:1767319311.248028:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319311.248078:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915617792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 86us (687us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319311.248085:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1366 00010000:00010000:1.0:1767319311.492913:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319312.517100:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319313.540627:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319314.372603:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319289/real 1767319289] req@ffff934633014380 x1853163982777856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319314 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319314.372683:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934633014380 x1853163982777856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319314 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319314.372716:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319314.372742:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319314.372747:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982777856:0@lo:250:kworker.0 00000100:00080000:3.0:1767319314.372816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319314.372823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319314.372828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319314.372834:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4538 00000100:00080000:3.0:1767319314.372838:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319314.372855:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319314.372890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163982783360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319314.372979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319314.372982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319314.372984:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319314.372988:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4558 00000100:00080000:3.0:1767319314.372999:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319314.373039:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319314.373055:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163982783616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319314.373071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319314.373074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319314.373076:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319314.373092:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4558 00000100:00080000:3.0:1767319314.373095:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319314.373099:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319314.373111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966a00 x1853163982783872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319314.373120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319314.373123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319314.373126:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319314.373130:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4558 00000100:00080000:3.0:1767319314.373133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319314.373137:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319314.373146:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982783360:0@lo:250:kworker.0 00000100:00100000:3.0:1767319314.373152:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163982784000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000100:0.0:1767319314.373199:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982783360 offset 0 length 520: 4 00000100:00100000:1.0:1767319314.373222:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982783488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319314.373245:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982783616:0@lo:38:kworker.0 00000100:00080000:3.0:1767319314.373255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319314.373260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319314.373263:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319314.373267:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4558 00000100:00080000:3.0:1767319314.373300:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319314.373304:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319314.373311:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319314.373320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9c00 x1853163982784384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319314.373324:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982784128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.373356:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783488 00000100:00100000:3.0:1767319314.373363:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783488 00000100:00100000:1.0:1767319314.373378:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319314.373389:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982783744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319314.373395:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319314.373421:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982784256:0@lo:400:kworker.0 00000100:00100000:1.0:1767319314.373424:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319314.373434:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040 00000100:00100000:3.0:1767319314.373446:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982783488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319314.373454:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784128 00000100:00100000:0.0:1767319314.373459:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784128 00000100:00100000:2.0:1767319314.373474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319314.373483:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.373504:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784256 00000100:00100000:0.0:1767319314.373508:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784256 00000100:00100000:1.0:1767319314.373515:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982783872:0@lo:38:kworker.0 00000100:00100000:2.0:1767319314.373521:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 657 00000100:00100000:3.0:1767319314.373525:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982783488:12345-0@lo:400:kworker.0 Request processed in 79us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319314.373535:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982784128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.373536:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040 00000100:00100000:0.0:1767319314.373543:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 658 00000100:00100000:1.0:1767319314.373549:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319314.373554:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982784256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.373561:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783744 00000100:00100000:3.0:1767319314.373566:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783744 00000100:00100000:1.0:1767319314.373580:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982784000:0@lo:38:kworker.0 00000100:00100000:2.0:1767319314.373589:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982784128:12345-0@lo:400:kworker.0 Request processed in 58us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319314.373591:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982784256:12345-0@lo:400:kworker.0 Request processed in 42us (118us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319314.373593:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783616 00000100:00100000:2.0:1767319314.373598:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 657 00000100:00100000:3.0:1767319314.373599:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783616 00000100:00100000:0.0:1767319314.373604:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 658 00000100:00100000:3.0:1767319314.373617:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982783872 00000100:00100000:1.0:1767319314.373619:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319314.373622:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982783872 00000100:00100000:0.0:1767319314.373630:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041 00000100:00100000:0.0:1767319314.373639:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982783744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319314.373640:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982784128:0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.373647:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1367 00000100:00100000:1.0:1767319314.373649:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982784384:0@lo:38:kworker.0 00000100:00100000:3.0:1767319314.373656:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982783616:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319314.373672:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982784256:0@lo:400:kworker.0 00000100:00100000:0.0:1767319314.373675:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982783744:12345-0@lo:400:kworker.0 Request processed in 38us (252us total) trans 0 rc 0/0 00010000:00020000:3.0:1767319314.373677:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319314.373689:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319314.373698:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319314.373705:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319314.373709:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319314.373725:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319314.373727:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982783616:12345-0@lo:38:kworker.0 Request processed in 70us (243us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767319314.373733:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319314.373738:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784384 00000100:00100000:1.0:1767319314.373739:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982783616:0@lo:38:kworker.0 00000100:00100000:2.0:1767319314.373745:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784384 00000100:00100000:0.0:1767319314.373907:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041 00000100:00100000:3.0:1767319314.379393:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1367 00000100:00100000:3.0:1767319314.379441:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1368 00000100:00100000:3.0:1767319314.379451:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982783872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319314.379474:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319314.381126:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1369 00000100:00100000:2.0:1767319314.381139:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982784384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319314.381178:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319314.381341:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982783872:12345-0@lo:38:kworker.0 Request processed in 1888us (7790us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319314.381351:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1368 00010000:00080000:3.0:1767319314.381415:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163982783744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319330 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319314.381428:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982783744:0@lo:400:kworker.0 00010000:00080000:3.0:1767319314.381440:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff480 x1853163982783488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319330 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319314.381447:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982783488:0@lo:400:kworker.0 00000100:00100000:3.0:1767319314.381511:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784000 00000100:00100000:2.0:1767319314.381833:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982784384:12345-0@lo:38:kworker.0 Request processed in 694us (8146us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319314.381842:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1369 00000100:00080000:2.0:1767319314.381997:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319314.382014:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319314.382018:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982784384:0@lo:38:kworker.0 00000100:00080000:2.0:1767319314.382034:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319314.382038:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319314.382040:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982783872:0@lo:38:kworker.0 00000100:00100000:3.0:1767319314.382489:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784000 00000100:00100000:3.0:1767319314.382786:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 367 00000100:00100000:3.0:1767319314.382793:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982784000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319314.382812:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319314.382886:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982784000:12345-0@lo:38:kworker.0 Request processed in 92us (9270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319314.382893:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 367 00000100:00080000:2.0:1767319314.384359:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319314.384371:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319314.384375:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982784000:0@lo:38:kworker.0 00010000:00010000:1.0:1767319314.564670:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319315.588535:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319315.908591:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982784512:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319315.908661:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319315.908765:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784512 00000100:00100000:0.0:1767319315.908773:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784512 00000100:00100000:0.0:1767319315.908831:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000100:00100000:0.0:1767319315.908840:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982784512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319315.908893:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319315.908957:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982784512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 117us (295us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319315.908964:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00000100:00100000:3.0:1767319315.909058:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982784512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319315.909656:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319315.909670:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982784640:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319315.909703:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319315.909764:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784640 00000100:00100000:1.0:1767319315.909776:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784640 00000100:00100000:1.0:1767319315.909832:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000100:00100000:1.0:1767319315.909843:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982784640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319315.909876:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319315.909951:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982784640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319315.909959:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00000100:00100000:0.0:1767319315.910066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982784640:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319316.370681:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319316.370743:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319316.370801:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618048 00000100:00100000:0.0:1767319316.370816:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618048 00000100:00100000:0.0:1767319316.370870:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618176 00000100:00100000:0.0:1767319316.370874:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618176 00000100:00100000:0.0:1767319316.370927:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1042 00000100:00100000:0.0:1767319316.370936:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915618176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319316.371053:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1370 00000100:00100000:0.0:1767319316.371059:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915618176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 121us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319316.371066:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915618048:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:0.0:1767319316.371073:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1042 00010000:00020000:3.0:1767319316.371092:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319316.371268:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915618048:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 204us (604us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319316.371383:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1370 00000100:00080000:1.0:1767319316.372508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319316.612598:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319317.636619:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319318.662386:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319319.492390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319319.492398:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319319.492451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319319.492455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319319.492458:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319319.492464:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4562 00000100:00080000:3.0:1767319319.492467:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319319.492473:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319319.492489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8a80 x1853163982784896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319319.492540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319319.492543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319319.492565:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319319.492569:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4562 00000100:00080000:3.0:1767319319.492572:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319319.492577:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319319.492681:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982785024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.492746:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319319.492775:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319319.492779:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982784768:0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.492794:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319319.492824:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785024 00000100:00100000:0.0:1767319319.492830:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785024 00000100:00100000:0.0:1767319319.492886:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784768 00000100:00100000:0.0:1767319319.492889:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784768 00000100:00100000:0.0:1767319319.492926:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043 00000100:00100000:0.0:1767319319.492935:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982785024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.492979:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982785024:12345-0@lo:400:kworker.0 Request processed in 46us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319319.492984:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043 00000100:00100000:0.0:1767319319.492996:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044 00000100:00100000:0.0:1767319319.492999:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982784768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.493014:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982784768:12345-0@lo:400:kworker.0 Request processed in 15us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319319.493018:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044 00010000:00080000:0.0:1767319319.493070:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a8e00 x1853163982784768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319335 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319319.493084:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982784768:0@lo:400:kworker.0 00010000:00080000:0.0:1767319319.493094:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0abb80 x1853163982785024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319335 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319319.493103:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982785024:0@lo:400:kworker.0 00000100:00080000:1.0:1767319319.493172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319319.493200:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982784896:0@lo:38:kworker.0 00000100:00100000:1.0:1767319319.493226:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319319.493341:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9180 x1853163982785152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319319.493357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319319.493359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319319.493361:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319319.493363:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4562 00000100:00080000:3.0:1767319319.493365:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319319.493367:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319319.493379:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aad80 x1853163982785280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319319.493405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319319.493408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319319.493409:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319319.493413:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4562 00000100:00080000:3.0:1767319319.493415:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319319.493418:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319319.493430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab480 x1853163982785664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319319.493488:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982785152:0@lo:38:kworker.0 00000100:00100000:1.0:1767319319.493526:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319319.493549:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982785280:0@lo:38:kworker.0 00000100:00100000:1.0:1767319319.493691:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319319.493712:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982785664:0@lo:38:kworker.0 00000100:00100000:1.0:1767319319.493736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319319.493810:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982784896 00000100:00100000:2.0:1767319319.493815:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982784896 00000100:00100000:2.0:1767319319.493832:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785152 00000100:00100000:2.0:1767319319.493834:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785152 00000100:00100000:2.0:1767319319.493840:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785664 00000100:00100000:2.0:1767319319.493842:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785664 00000100:00100000:2.0:1767319319.493851:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1371 00000100:00100000:2.0:1767319319.493856:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982784896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319319.493888:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319319.493938:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982784896:12345-0@lo:38:kworker.0 Request processed in 81us (709us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319319.493944:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1371 00000100:00100000:2.0:1767319319.493956:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1372 00000100:00100000:2.0:1767319319.493960:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982785152:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319319.493969:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319319.493990:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982785152:12345-0@lo:38:kworker.0 Request processed in 28us (464us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319319.493995:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1372 00000100:00100000:2.0:1767319319.494005:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1373 00000100:00100000:2.0:1767319319.494008:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982785664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319319.494029:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319319.494052:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982785664:12345-0@lo:38:kworker.0 Request processed in 42us (315us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319319.494057:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1373 00000100:00100000:2.0:1767319319.494078:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982785536:0@lo:400:kworker.0 00000100:00100000:2.0:1767319319.494102:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319319.494120:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319319.494125:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982785408:0@lo:400:kworker.0 00000100:00100000:2.0:1767319319.494141:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319319.494265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319319.494292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319319.494296:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982785152:0@lo:38:kworker.0 00000100:00080000:1.0:1767319319.494306:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319319.494309:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319319.494312:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982785664:0@lo:38:kworker.0 00000100:00080000:1.0:1767319319.494319:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319319.494322:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319319.494325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982784896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319319.494410:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785280 00000100:00100000:3.0:1767319319.494414:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785280 00000100:00100000:3.0:1767319319.494443:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 368 00000100:00100000:3.0:1767319319.494449:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982785280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319319.494459:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319319.494499:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982785280:12345-0@lo:38:kworker.0 Request processed in 49us (915us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319319.494506:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 368 00000100:00100000:3.0:1767319319.494543:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785536 00000100:00100000:3.0:1767319319.494546:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785536 00000100:00080000:0.0:1767319319.494684:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319319.494713:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785408 00000100:00100000:0.0:1767319319.494717:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785408 00000100:00100000:0.0:1767319319.494746:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 659 00000100:00100000:0.0:1767319319.494767:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982785408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319319.494806:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982785408:12345-0@lo:400:kworker.0 Request processed in 41us (664us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319319.494813:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 659 00000100:00080000:0.0:1767319319.494830:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319319.494922:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 660 00000100:00100000:3.0:1767319319.494930:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982785536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319319.494961:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982785536:12345-0@lo:400:kworker.0 Request processed in 32us (858us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319319.494967:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 660 00000100:00080000:1.0:1767319319.495036:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319319.495040:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319319.495043:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982785280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319319.495154:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982785408:0@lo:400:kworker.0 00000100:00100000:2.0:1767319319.495164:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982785536:0@lo:400:kworker.0 00010000:00010000:1.0:1767319319.684506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319320.712863:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319321.031107:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982785920:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319321.031228:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319321.031423:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319321.031431:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982785792:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319321.031472:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319321.036478:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785920 00000100:00100000:1.0:1767319321.036495:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785920 00000100:00100000:1.0:1767319321.036714:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982785792 00000100:00100000:1.0:1767319321.036719:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982785792 00000100:00100000:3.0:1767319321.036808:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 712 00000100:00100000:3.0:1767319321.036835:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982785920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319321.036876:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319321.036988:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982785920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 168us (5759us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319321.036995:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 712 00000100:00100000:3.0:1767319321.037059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982785920:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319321.044295:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319321.044307:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319321.046315:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 713 00000100:00100000:1.0:1767319321.046331:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982785792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319321.046368:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319321.046465:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982785792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 140us (14991us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319321.046473:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 713 00000100:00100000:3.0:1767319321.046654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982785792:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319321.500831:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319321.500899:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319321.500992:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618304 00000100:00100000:3.0:1767319321.501003:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618304 00000100:00100000:3.0:1767319321.501047:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618432 00000100:00100000:3.0:1767319321.501050:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618432 00000100:00100000:3.0:1767319321.501100:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1045 00000100:00100000:3.0:1767319321.501110:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915618432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319321.501181:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915618432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319321.501189:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1045 00000100:00100000:3.0:1767319321.501208:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1374 00000100:00100000:3.0:1767319321.501213:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915618304:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319321.501232:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319321.501255:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915618304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (433us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319321.501262:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1374 00000100:00080000:1.0:1767319321.501425:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319321.732820:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319322.756468:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319323.781354:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319324.613296:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319324.613306:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319324.613377:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319324.613381:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319324.613385:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319324.613392:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4568 00000100:00080000:3.0:1767319324.613397:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319324.613404:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319324.613427:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab100 x1853163982786176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319324.613451:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319324.613453:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319324.613454:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319324.613458:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4568 00000100:00080000:3.0:1767319324.613460:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319324.613463:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319324.613474:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8380 x1853163982786432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319324.613481:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319324.613483:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319324.613485:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319324.613488:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4568 00000100:00080000:3.0:1767319324.613490:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319324.613493:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319324.613503:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8000 x1853163982786560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319324.613537:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319324.613540:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319324.613542:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319324.613546:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4568 00000100:00080000:3.0:1767319324.613548:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319324.613551:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319324.613561:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aad80 x1853163982786944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319324.613589:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982786688:0@lo:400:kworker.0 00000100:00100000:3.0:1767319324.613654:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.613685:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982786048:0@lo:400:kworker.0 00000100:00100000:3.0:1767319324.613690:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319324.613695:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982786816:0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.613717:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319324.613726:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.613742:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319324.613747:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982786304:0@lo:400:kworker.0 00000100:00100000:3.0:1767319324.613757:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786688 00000100:00100000:3.0:1767319324.613763:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786688 00000100:00100000:0.0:1767319324.613781:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319324.613805:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786816 00000100:00100000:0.0:1767319324.613809:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786816 00000100:00100000:3.0:1767319324.613827:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 661 00000100:00100000:3.0:1767319324.613836:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982786688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319324.613847:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 662 00000100:00100000:0.0:1767319324.613852:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982786816:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319324.613938:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319324.613942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319324.613957:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982786176:0@lo:38:kworker.0 00000100:00100000:1.0:1767319324.613982:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.614013:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982786432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319324.614030:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.614043:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982786560:0@lo:38:kworker.0 00000100:00100000:1.0:1767319324.614080:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.614120:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982786944:0@lo:38:kworker.0 00000100:00100000:1.0:1767319324.614140:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319324.614166:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786048 00000100:00100000:1.0:1767319324.614170:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786048 00000100:00100000:1.0:1767319324.614195:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786304 00000100:00100000:1.0:1767319324.614198:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786304 00000100:00100000:1.0:1767319324.614207:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786176 00000100:00100000:1.0:1767319324.614209:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786176 00000100:00100000:1.0:1767319324.614216:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786432 00000100:00100000:1.0:1767319324.614218:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786432 00000100:00100000:1.0:1767319324.614224:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786944 00000100:00100000:1.0:1767319324.614226:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786944 00000100:00100000:1.0:1767319324.614234:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046 00000100:00100000:1.0:1767319324.614240:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982786048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319324.614287:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982786048:12345-0@lo:400:kworker.0 Request processed in 32us (554us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319324.614294:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046 00000100:00100000:1.0:1767319324.614309:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047 00000100:00100000:1.0:1767319324.614314:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982786304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319324.614340:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982786304:12345-0@lo:400:kworker.0 Request processed in 26us (557us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319324.614344:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047 00000100:00100000:1.0:1767319324.614355:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1375 00000100:00100000:1.0:1767319324.614358:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319324.614374:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319324.614403:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786176:12345-0@lo:38:kworker.0 Request processed in 43us (419us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319324.614408:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1375 00000100:00100000:1.0:1767319324.614419:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1376 00000100:00100000:1.0:1767319324.614422:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319324.614430:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319324.614451:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786432:12345-0@lo:38:kworker.0 Request processed in 27us (420us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319324.614456:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1376 00000100:00100000:1.0:1767319324.614466:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1377 00000100:00100000:1.0:1767319324.614470:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319324.614478:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319324.614500:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982786944:12345-0@lo:38:kworker.0 Request processed in 28us (359us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319324.614504:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1377 00000100:00100000:1.0:1767319324.614522:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982786560 00000100:00100000:1.0:1767319324.614525:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982786560 00000100:00100000:1.0:1767319324.614588:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 369 00000100:00100000:1.0:1767319324.614592:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982786560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319324.614600:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319324.614626:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982786560:12345-0@lo:38:kworker.0 Request processed in 33us (545us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319324.614631:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 369 00000100:00080000:1.0:1767319324.614651:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319324.614674:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.614678:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.614680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982786176:0@lo:38:kworker.0 00000100:00080000:1.0:1767319324.614689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.614692:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.614694:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982786432:0@lo:38:kworker.0 00000100:00080000:1.0:1767319324.614702:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.614704:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.614706:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982786560:0@lo:38:kworker.0 00000100:00080000:1.0:1767319324.614712:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319324.614715:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319324.614717:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982786944:0@lo:38:kworker.0 00010000:00080000:1.0:1767319324.614785:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9f80 x1853163982786304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319340 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319324.614796:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982786304:0@lo:400:kworker.0 00010000:00080000:1.0:1767319324.614804:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aaa00 x1853163982786048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319340 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319324.614811:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982786048:0@lo:400:kworker.0 00000100:00100000:3.0:1767319324.615152:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982786688:12345-0@lo:400:kworker.0 Request processed in 1314us (1495us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319324.615160:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 661 00000100:00100000:0.0:1767319324.627696:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982786816:12345-0@lo:400:kworker.0 Request processed in 13821us (13945us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319324.627714:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 662 00000100:00100000:3.0:1767319324.634081:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982786816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319324.634106:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982786688:0@lo:400:kworker.0 00010000:00010000:1.0:1767319324.807383:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319325.832267:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319326.148706:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982787200:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319326.148827:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319326.148922:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982787072:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319326.149165:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787200 00000100:00100000:0.0:1767319326.149173:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787200 00000100:00100000:2.0:1767319326.149205:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319326.149245:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787072 00000100:00100000:0.0:1767319326.149249:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787072 00000100:00100000:0.0:1767319326.149296:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000100:00100000:0.0:1767319326.149306:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982787200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319326.149357:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319326.149411:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982787200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (583us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319326.149419:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00100000:0.0:1767319326.149449:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 715 00000100:00100000:0.0:1767319326.149455:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982787072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319326.149471:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319326.149503:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982787072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319326.149509:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 715 00000100:00100000:0.0:1767319326.149550:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982787072:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319326.149682:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319326.149688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319326.152916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982787200:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319326.612619:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319326.612866:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319326.624882:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618688 00000100:00100000:2.0:1767319326.624899:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618688 00000100:00100000:2.0:1767319326.625019:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618560 00000100:00100000:2.0:1767319326.625023:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618560 00000100:00100000:2.0:1767319326.625047:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1048 00000100:00100000:2.0:1767319326.625058:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915618688:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319326.625898:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915618688:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 838us (13286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319326.625908:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1048 00000100:00100000:2.0:1767319326.625944:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1378 00000100:00100000:2.0:1767319326.625950:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915618560:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319326.625972:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319326.626009:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915618560:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (13144us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319326.626025:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1378 00000100:00080000:1.0:1767319326.626170:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319326.852988:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319327.876479:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319328.900632:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319329.732576:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319329.732586:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319329.732685:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319329.732690:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319329.732693:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319329.732701:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4573 00000100:00080000:3.0:1767319329.732706:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319329.732714:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319329.732747:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9180 x1853163982787456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319329.732783:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319329.732786:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319329.732788:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319329.732792:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4573 00000100:00080000:3.0:1767319329.732794:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319329.732798:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319329.732808:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8e00 x1853163982787712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319329.732818:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319329.732821:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319329.732823:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319329.732841:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4573 00000100:00080000:3.0:1767319329.732844:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319329.732847:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319329.732874:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9c00 x1853163982787840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319329.732920:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319329.732923:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319329.732925:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319329.732929:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4573 00000100:00080000:3.0:1767319329.732932:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319329.732936:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319329.732963:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9f80 x1853163982788224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319329.733107:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982787584:0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.733195:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319329.733298:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319329.733305:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982787328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.733329:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319329.733370:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787584 00000100:00100000:1.0:1767319329.733378:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787584 00000100:00100000:1.0:1767319329.733451:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787328 00000100:00100000:1.0:1767319329.733454:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787328 00000100:00100000:1.0:1767319329.733479:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049 00000100:00100000:1.0:1767319329.733487:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982787584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.733538:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982787584:12345-0@lo:400:kworker.0 Request processed in 51us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319329.733546:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049 00000100:00100000:1.0:1767319329.733584:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050 00000100:00100000:1.0:1767319329.733590:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982787328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.733618:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982787328:12345-0@lo:400:kworker.0 Request processed in 29us (288us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319329.733623:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050 00000100:00080000:1.0:1767319329.733640:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319329.733674:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9880 x1853163982787328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319345 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319329.733688:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982787328:0@lo:400:kworker.0 00010000:00080000:1.0:1767319329.733698:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0abb80 x1853163982787584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319345 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319329.733705:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982787584:0@lo:400:kworker.0 00000100:00100000:0.0:1767319329.733813:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982787968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319329.733846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319329.733879:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319329.733885:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982788096:0@lo:400:kworker.0 00000100:00100000:0.0:1767319329.733916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319329.734051:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787968 00000100:00100000:1.0:1767319329.734056:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787968 00000100:00100000:1.0:1767319329.734099:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788096 00000100:00100000:1.0:1767319329.734102:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788096 00000100:00100000:1.0:1767319329.734122:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 663 00000100:00100000:1.0:1767319329.734129:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982787968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.734165:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982787968:12345-0@lo:400:kworker.0 Request processed in 37us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319329.734171:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 663 00000100:00100000:1.0:1767319329.734182:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 664 00000100:00100000:1.0:1767319329.734187:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982788096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319329.734212:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982788096:12345-0@lo:400:kworker.0 Request processed in 25us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319329.734217:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 664 00000100:00080000:0.0:1767319329.734308:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319329.734312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319329.734332:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982788096:0@lo:400:kworker.0 00000100:00100000:0.0:1767319329.734342:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982787968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319329.736381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982787456:0@lo:38:kworker.0 00000100:00100000:2.0:1767319329.736484:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319329.736543:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982787712:0@lo:38:kworker.0 00000100:00100000:2.0:1767319329.736568:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319329.736599:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982787840:0@lo:38:kworker.0 00000100:00100000:1.0:1767319329.736635:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787456 00000100:00100000:2.0:1767319329.736643:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319329.736644:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787456 00000100:00100000:1.0:1767319329.736686:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787712 00000100:00100000:2.0:1767319329.736689:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982788224:0@lo:38:kworker.0 00000100:00100000:1.0:1767319329.736691:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787712 00000100:00100000:1.0:1767319329.736715:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1379 00000100:00100000:1.0:1767319329.736741:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982787456:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319329.736742:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767319329.736767:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319329.736784:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788224 00000100:00100000:2.0:1767319329.736789:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788224 00000100:00100000:2.0:1767319329.736811:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1380 00000100:00100000:2.0:1767319329.736817:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982787712:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319329.736824:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982787456:12345-0@lo:38:kworker.0 Request processed in 82us (338us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319329.736834:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1379 00010000:00020000:2.0:1767319329.736850:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319329.736854:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1381 00000100:00100000:1.0:1767319329.736860:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319329.736869:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319329.736888:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982787712:12345-0@lo:38:kworker.0 Request processed in 69us (318us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319329.736896:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1380 00000100:00080000:2.0:1767319329.737431:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319329.737437:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319329.737441:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982787456:0@lo:38:kworker.0 00000100:00100000:0.0:1767319329.737530:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982787840 00000100:00100000:0.0:1767319329.737535:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982787840 00000100:00100000:0.0:1767319329.737569:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 370 00000100:00100000:0.0:1767319329.737575:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982787840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319329.737587:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319329.737746:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788224:12345-0@lo:38:kworker.0 Request processed in 882us (1004us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319329.737754:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1381 00000100:00080000:2.0:1767319329.738058:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319329.738063:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319329.738067:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982787712:0@lo:38:kworker.0 00000100:00080000:2.0:1767319329.738077:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319329.738081:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319329.738084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982788224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319329.740660:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982787840:12345-0@lo:38:kworker.0 Request processed in 3072us (4006us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319329.740677:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 370 00000100:00080000:2.0:1767319329.740777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319329.740782:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319329.740786:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982787840:0@lo:38:kworker.0 00010000:00010000:1.0:1767319329.926182:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319330.948752:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319331.268666:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319331.268685:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982788352:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319331.268787:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319331.269014:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319331.269025:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982788480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319331.269055:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319331.269165:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788352 00000100:00100000:0.0:1767319331.269175:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788352 00000100:00100000:0.0:1767319331.269245:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788480 00000100:00100000:0.0:1767319331.269249:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788480 00000100:00100000:0.0:1767319331.269268:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00100000:0.0:1767319331.269298:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982788352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319331.269329:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319331.269376:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982788352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (588us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319331.269383:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00000100:00100000:0.0:1767319331.269401:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 717 00000100:00100000:0.0:1767319331.269408:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982788480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319331.269426:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319331.269458:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982788480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (402us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319331.269464:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 717 00000100:00100000:0.0:1767319331.269504:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982788480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319331.269537:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982788352:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319331.269633:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319331.269637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319331.724451:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319331.726641:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618944 00000100:00100000:1.0:1767319331.726655:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618944 00000100:00100000:1.0:1767319331.726769:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1051 00000100:00100000:1.0:1767319331.726781:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915618944:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319331.726891:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915618944:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 112us (2449us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319331.726899:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1051 00000100:00080000:2.0:1767319331.727590:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319331.728876:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319331.729035:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915618816 00000100:00100000:3.0:1767319331.729046:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915618816 00000100:00100000:3.0:1767319331.729092:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1382 00000100:00100000:3.0:1767319331.729108:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915618816:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319331.729132:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319331.729202:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915618816:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (333us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319331.729211:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1382 00010000:00010000:1.0:1767319331.972588:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319332.997076:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319334.022762:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319334.854930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319334.854985:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319334.855054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319334.855058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319334.855062:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319334.855069:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4578 00000100:00080000:3.0:1767319334.855082:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319334.855111:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319334.855130:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982788608:0@lo:400:kworker.0 00000100:00100000:1.0:1767319334.855199:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319334.855340:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788608 00000100:00100000:1.0:1767319334.855348:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788608 00000100:00100000:1.0:1767319334.855408:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052 00000100:00100000:1.0:1767319334.855418:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982788608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319334.855567:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982788608:12345-0@lo:400:kworker.0 Request processed in 151us (368us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319334.855575:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052 00000100:00080000:2.0:1767319334.855652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319334.856961:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319334.856999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8380 x1853163982788736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319334.857046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319334.857051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319334.857054:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319334.857059:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4578 00000100:00080000:3.0:1767319334.857062:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319334.857067:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319334.857075:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982788736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319334.857081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9c00 x1853163982788992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319334.857104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319334.857108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319334.857110:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319334.857114:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4578 00000100:00080000:3.0:1767319334.857117:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319334.857121:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319334.857132:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319334.857136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9f80 x1853163982789120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319334.857185:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982788992:0@lo:38:kworker.0 00000100:00080000:3.0:1767319334.857204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319334.857211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319334.857212:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319334.857215:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319334.857220:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4578 00000100:00080000:3.0:1767319334.857222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319334.857225:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319334.857237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab800 x1853163982789504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319334.857258:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982789120:0@lo:38:kworker.0 00000100:00100000:3.0:1767319334.857290:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982789248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319334.857340:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319334.857351:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982788864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.865924:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319334.866044:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319334.866057:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982789376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.866102:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319334.866147:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789248 00000100:00100000:3.0:1767319334.866154:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789248 00000100:00100000:3.0:1767319334.866205:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789376 00000100:00100000:3.0:1767319334.866208:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789376 00000100:00100000:3.0:1767319334.866227:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 665 00000100:00100000:3.0:1767319334.866234:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982789248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.866304:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982789248:12345-0@lo:400:kworker.0 Request processed in 70us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319334.866311:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 665 00000100:00100000:3.0:1767319334.866336:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 666 00000100:00100000:3.0:1767319334.866342:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982789376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.866365:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982789376:12345-0@lo:400:kworker.0 Request processed in 24us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319334.866370:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 666 00000100:00080000:3.0:1767319334.866391:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319334.866395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319334.866420:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982789376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.866429:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982789248:0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.866470:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788736 00000100:00100000:3.0:1767319334.866473:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788736 00000100:00100000:3.0:1767319334.866490:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788992 00000100:00100000:3.0:1767319334.866493:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788992 00000100:00100000:3.0:1767319334.866503:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1383 00000100:00100000:3.0:1767319334.866509:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319334.866525:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319334.866553:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788736:12345-0@lo:38:kworker.0 Request processed in 43us (9422us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319334.866558:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1383 00000100:00100000:3.0:1767319334.866567:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1384 00000100:00100000:3.0:1767319334.866570:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319334.866577:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319334.866596:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982788992:12345-0@lo:38:kworker.0 Request processed in 24us (9386us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319334.866600:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1384 00000100:00100000:1.0:1767319334.876156:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319334.876315:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982788864 00000100:00100000:3.0:1767319334.876326:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982788864 00000100:00100000:3.0:1767319334.876413:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053 00000100:00100000:3.0:1767319334.876424:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982788864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.876488:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982788864:12345-0@lo:400:kworker.0 Request processed in 66us (339us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319334.876495:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053 00000100:00080000:1.0:1767319334.876894:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319334.876928:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0ab100 x1853163982788864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319350 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319334.876943:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982788864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319334.883787:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319334.883926:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319334.883933:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319334.883938:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982788736:0@lo:38:kworker.0 00000100:00100000:2.0:1767319334.883971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982789504:0@lo:38:kworker.0 00000100:00100000:2.0:1767319334.884031:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319334.884063:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319334.884068:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319334.884071:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982788992:0@lo:38:kworker.0 00000100:00100000:1.0:1767319334.884081:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789120 00000100:00100000:1.0:1767319334.884088:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789120 00000100:00100000:1.0:1767319334.884130:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 371 00000100:00100000:1.0:1767319334.884138:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982789120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319334.884159:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319334.884208:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982789120:12345-0@lo:38:kworker.0 Request processed in 69us (439us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319334.884214:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 371 00000100:00080000:2.0:1767319334.884375:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319334.884380:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319334.884383:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982789120:0@lo:38:kworker.0 00010000:00080000:0.0:1767319334.885197:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a8000 x1853163982788608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319350 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319334.885208:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982788608:0@lo:400:kworker.0 00000100:00100000:3.0:1767319334.885609:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789504 00000100:00100000:3.0:1767319334.885616:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789504 00000100:00100000:3.0:1767319334.885690:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1385 00000100:00100000:3.0:1767319334.885697:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982789504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319334.885714:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319334.885791:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982789504:12345-0@lo:38:kworker.0 Request processed in 93us (1759us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319334.885798:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1385 00000100:00080000:2.0:1767319334.885832:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319334.885838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319334.885841:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982789504:0@lo:38:kworker.0 00010000:00010000:1.0:1767319335.048094:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319335.822033:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915619072 offset 0 length 520: 4 00000100:00100000:3.0:1767319335.822094:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319335.822170:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619200 00000100:00100000:1.0:1767319335.822180:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619200 00000100:00100000:1.0:1767319335.822226:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1386 00000100:00100000:1.0:1767319335.822234:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915619200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319335.822257:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319335.822437:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915619200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 201us (341us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319335.822445:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1386 00000100:00100000:2.0:1767319335.828350:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319335.828782:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619328 00000100:00100000:1.0:1767319335.828792:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619328 00000100:00080000:2.0:1767319335.828931:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319335.828978:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1054 00000100:00100000:1.0:1767319335.828990:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915619328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319335.829069:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915619328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (729us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319335.829078:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1054 00010000:00010000:1.0:1767319336.068476:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319336.388739:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982789760:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319336.388814:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319336.388901:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319336.388909:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982789632:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319336.389464:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789760 00000100:00100000:1.0:1767319336.389475:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789760 00000100:00100000:1.0:1767319336.389550:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 718 00000100:00100000:1.0:1767319336.389561:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982789760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319336.389633:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319336.389685:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982789760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 126us (870us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319336.389692:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 718 00000100:00100000:3.0:1767319336.390368:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319336.390433:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982789760:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319336.390538:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319336.390657:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982789632 00000100:00100000:1.0:1767319336.390663:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982789632 00000100:00100000:1.0:1767319336.390714:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00100000:1.0:1767319336.390722:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982789632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319336.390764:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319336.390817:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982789632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (451us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319336.390824:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00080000:0.0:1767319336.390873:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319336.390944:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982789632:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319337.092486:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319338.116528:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319339.012637:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319314/real 1767319314] req@ffff934638595500 x1853163982783360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319339 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319339.012675:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934638595500 x1853163982783360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319339 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319339.012697:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319339.012723:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319339.012727:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982783360:0@lo:250:kworker.0 00000100:00080000:0.0:1767319339.013139:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319339.013145:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319339.013148:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319339.013153:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4562 00000100:00080000:0.0:1767319339.013157:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319339.013164:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319339.013196:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9500 x1853163982789888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319339.013250:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319339.013254:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319339.013256:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319339.013260:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4583 00000100:00080000:0.0:1767319339.013262:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319339.013266:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319339.013296:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163982790144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319339.013300:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982790016:0@lo:400:kworker.0 00000100:00080000:0.0:1767319339.013312:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319339.013314:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319339.013316:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319339.013332:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4583 00000100:00080000:0.0:1767319339.013334:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319339.013337:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319339.013347:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9f80 x1853163982790400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319339.013354:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319339.013356:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319339.013358:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319339.013361:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4583 00000100:00080000:0.0:1767319339.013363:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319339.013363:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319339.013367:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319339.013377:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163982790528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319339.013406:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319339.013410:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319339.013411:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319339.013415:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4583 00000100:00080000:0.0:1767319339.013417:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319339.013421:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319339.013429:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319339.013432:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb480 x1853163982790912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319339.013435:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982790272:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.013456:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982790656:0@lo:400:kworker.0 00000100:00100000:1.0:1767319339.013457:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319339.013485:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319339.013485:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790016 00000100:00100000:1.0:1767319339.013491:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790016 00000100:00100000:0.0:1767319339.013515:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319339.013520:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982790784:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.013541:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319339.013546:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790272 00000100:00100000:1.0:1767319339.013558:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790272 00000100:00100000:1.0:1767319339.013581:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055 00000100:00100000:1.0:1767319339.013589:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982790016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319339.013622:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982789888:0@lo:250:kworker.0 00000100:00100000:1.0:1767319339.013643:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982790016:12345-0@lo:400:kworker.0 Request processed in 54us (279us total) trans 0 rc 0/0 00000400:00000100:2.0:1767319339.013643:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982789888 offset 0 length 520: 4 00000100:00100000:2.0:1767319339.013665:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982790144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.013686:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.013703:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982790400:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.013722:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.013731:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982790528:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.013753:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.013768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982790912:0@lo:38:kworker.0 00000100:00100000:2.0:1767319339.013808:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319339.013833:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790144 00000100:00100000:2.0:1767319339.013850:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790144 00000100:00100000:3.0:1767319339.014084:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790656 00000100:00100000:3.0:1767319339.014088:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790656 00000100:00100000:3.0:1767319339.014120:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790784 00000100:00100000:3.0:1767319339.014123:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790784 00000100:00100000:3.0:1767319339.014141:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 667 00000100:00100000:3.0:1767319339.014148:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982790656:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.014190:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982790656:12345-0@lo:400:kworker.0 Request processed in 43us (704us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319339.014196:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 667 00000100:00100000:3.0:1767319339.014213:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 668 00000100:00100000:3.0:1767319339.014219:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982790784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319339.014246:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982790784:12345-0@lo:400:kworker.0 Request processed in 27us (704us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319339.014252:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 668 00000100:00100000:3.0:1767319339.014290:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790400 00000100:00100000:3.0:1767319339.014294:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790400 00000100:00100000:0.0:1767319339.014588:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056 00000100:00100000:0.0:1767319339.014598:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982790272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.014629:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982790272:12345-0@lo:400:kworker.0 Request processed in 35us (1172us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319339.014635:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056 00000100:00100000:0.0:1767319339.014649:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790912 00000100:00100000:0.0:1767319339.014652:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790912 00000100:00100000:0.0:1767319339.014663:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1387 00000100:00100000:0.0:1767319339.014667:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982790912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319339.014687:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319339.014718:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982790912:12345-0@lo:38:kworker.0 Request processed in 50us (910us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319339.014724:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1387 00000100:00080000:0.0:1767319339.014740:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319339.014744:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319339.014746:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319339.014771:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982790784:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.014780:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982790656:0@lo:400:kworker.0 00010000:00080000:0.0:1767319339.014819:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb100 x1853163982790272/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319355 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319339.014830:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982790272:0@lo:400:kworker.0 00010000:00080000:0.0:1767319339.014852:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8700 x1853163982790016/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319355 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319339.014860:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982790016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319339.014902:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982790528 00000100:00100000:0.0:1767319339.014905:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982790528 00000100:00100000:0.0:1767319339.014923:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 372 00000100:00100000:0.0:1767319339.014927:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982790528:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319339.014936:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319339.014963:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982790528:12345-0@lo:38:kworker.0 Request processed in 34us (1210us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319339.014969:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 372 00000100:00080000:0.0:1767319339.014991:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319339.014996:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319339.014999:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982790528:0@lo:38:kworker.0 00000100:00080000:0.0:1767319339.015005:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319339.015008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319339.015011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982790912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319339.016858:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055 00000100:00100000:2.0:1767319339.023054:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1388 00000100:00100000:2.0:1767319339.023066:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982790144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319339.023092:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319339.023188:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982790144:12345-0@lo:38:kworker.0 Request processed in 122us (9500us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319339.023196:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1388 00000100:00080000:0.0:1767319339.023522:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319339.023529:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319339.023533:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982790144:0@lo:38:kworker.0 00000100:00100000:3.0:1767319339.023783:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1389 00000100:00100000:3.0:1767319339.023791:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982790400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319339.023804:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319339.023863:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982790400:12345-0@lo:38:kworker.0 Request processed in 72us (10139us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319339.023870:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1389 00000100:00080000:0.0:1767319339.023924:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319339.023930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319339.023933:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982790400:0@lo:38:kworker.0 00010000:00010000:1.0:1767319339.140491:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319340.167283:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319340.945393:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319340.945444:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319340.945548:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619584 00000100:00100000:2.0:1767319340.945560:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619584 00000100:00100000:2.0:1767319340.945664:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619456 00000100:00100000:2.0:1767319340.945667:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619456 00000100:00100000:2.0:1767319340.945687:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1057 00000100:00100000:2.0:1767319340.945698:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915619584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319340.945798:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915619584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (415us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319340.945809:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1057 00000100:00100000:2.0:1767319340.945828:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1390 00000100:00100000:2.0:1767319340.945833:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915619456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319340.945854:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319340.945881:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915619456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (436us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319340.945887:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1390 00000100:00080000:3.0:1767319340.946344:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319341.188542:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319341.509632:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319341.509655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982791040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319341.509721:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319341.510536:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791040 00000100:00100000:3.0:1767319341.510546:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791040 00000100:00100000:3.0:1767319341.510606:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 00000100:00100000:3.0:1767319341.510614:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982791040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319341.510655:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319341.510704:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982791040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (982us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319341.510710:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00000100:00080000:1.0:1767319341.510915:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319341.521759:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982791040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319341.521799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982791168:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319341.521924:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319341.523027:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791168 00000100:00100000:0.0:1767319341.523040:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791168 00000100:00100000:0.0:1767319341.523154:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 721 00000100:00100000:0.0:1767319341.523164:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982791168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319341.523199:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319341.523285:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982791168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 108us (1346us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319341.523292:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 721 00000100:00100000:2.0:1767319341.525260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982791168:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319341.525363:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319342.212460:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319343.236546:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319344.068562:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319344.068572:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319344.068632:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319344.068636:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319344.068639:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319344.068647:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4587 00000100:00080000:2.0:1767319344.068651:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319344.068658:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319344.068682:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0a80 x1853163982791424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319344.068708:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319344.068711:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319344.068714:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319344.068717:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4587 00000100:00080000:2.0:1767319344.068720:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319344.068724:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319344.068734:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163982791680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319344.068742:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319344.068745:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319344.068747:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319344.068750:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4587 00000100:00080000:2.0:1767319344.068752:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319344.068756:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319344.068767:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2d80 x1853163982791808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319344.068820:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319344.068824:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319344.068825:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319344.068835:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4587 00000100:00080000:2.0:1767319344.068837:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319344.068841:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319344.068852:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163982792192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319344.068877:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982792064:0@lo:400:kworker.0 00000100:00100000:3.0:1767319344.068938:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982791936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319344.068942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.069008:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982791552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.069204:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982791296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.072356:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.072465:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791552 00000100:00100000:1.0:1767319344.072475:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791552 00000100:00100000:1.0:1767319344.072535:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058 00000100:00100000:1.0:1767319344.072545:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982791552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.072619:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982791552:12345-0@lo:400:kworker.0 Request processed in 76us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319344.072626:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058 00000100:00080000:1.0:1767319344.072652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319344.072679:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3b80 x1853163982791552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319360 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319344.072692:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982791552:0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.072737:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982791424:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.072772:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.072789:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982791680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.072824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.072848:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982791808:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.072871:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.072888:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982792192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.072911:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.072939:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791424 00000100:00100000:1.0:1767319344.072944:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791424 00000100:00100000:1.0:1767319344.072963:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791680 00000100:00100000:1.0:1767319344.072967:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791680 00000100:00100000:1.0:1767319344.072980:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792192 00000100:00100000:1.0:1767319344.072983:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792192 00000100:00100000:1.0:1767319344.072994:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1391 00000100:00100000:1.0:1767319344.072999:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982791424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319344.073016:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319344.073050:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982791424:12345-0@lo:38:kworker.0 Request processed in 50us (277us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319344.073057:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1391 00000100:00100000:1.0:1767319344.073068:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1392 00000100:00100000:1.0:1767319344.073072:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982791680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319344.073080:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319344.073101:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982791680:12345-0@lo:38:kworker.0 Request processed in 28us (277us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319344.073106:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1392 00000100:00100000:1.0:1767319344.073115:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1393 00000100:00100000:1.0:1767319344.073120:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982792192:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319344.073128:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319344.073149:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982792192:12345-0@lo:38:kworker.0 Request processed in 27us (237us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319344.073154:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1393 00000100:00080000:1.0:1767319344.073179:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319344.073183:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319344.073187:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982791424:0@lo:38:kworker.0 00000100:00080000:1.0:1767319344.073195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319344.073198:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319344.073200:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982791680:0@lo:38:kworker.0 00000100:00080000:1.0:1767319344.073207:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319344.073211:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319344.073213:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982792192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319344.073248:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791808 00000100:00100000:1.0:1767319344.073252:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791808 00000100:00100000:1.0:1767319344.073295:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 373 00000100:00100000:1.0:1767319344.073300:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982791808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319344.073311:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319344.073342:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982791808:12345-0@lo:38:kworker.0 Request processed in 40us (469us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319344.073347:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 373 00000100:00100000:1.0:1767319344.073365:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792064 00000100:00100000:1.0:1767319344.073370:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792064 00000100:00100000:1.0:1767319344.073401:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 669 00000100:00100000:1.0:1767319344.073408:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982792064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.073441:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982792064:12345-0@lo:400:kworker.0 Request processed in 34us (4502us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319344.073447:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 669 00000100:00080000:1.0:1767319344.073474:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319344.073489:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319344.073493:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319344.073496:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982791808:0@lo:38:kworker.0 00000100:00100000:2.0:1767319344.074033:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982792064:0@lo:400:kworker.0 00000100:00100000:0.0:1767319344.074141:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.074222:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791296 00000100:00100000:1.0:1767319344.074226:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791296 00000100:00100000:1.0:1767319344.074284:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059 00000100:00100000:1.0:1767319344.074292:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982791296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.074332:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982791296:12345-0@lo:400:kworker.0 Request processed in 41us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319344.074339:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059 00000100:00100000:3.0:1767319344.074350:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319344.074388:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982791936 00000100:00100000:1.0:1767319344.074393:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982791936 00000100:00100000:1.0:1767319344.074717:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 670 00000100:00100000:1.0:1767319344.074726:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982791936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319344.074770:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982791936:12345-0@lo:400:kworker.0 Request processed in 47us (422us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319344.074776:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 670 00000100:00100000:2.0:1767319344.074865:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982791936:0@lo:400:kworker.0 00000100:00080000:0.0:1767319344.075052:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319344.075057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319344.075077:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0e00 x1853163982791296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319360 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319344.075094:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982791296:0@lo:400:kworker.0 00010000:00010000:1.0:1767319344.260546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319345.284592:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319346.066497:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319346.066733:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619840 00000100:00100000:1.0:1767319346.066746:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619840 00000100:00100000:1.0:1767319346.066887:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1060 00000100:00100000:1.0:1767319346.066899:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915619840:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319346.066978:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915619840:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (496us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319346.066986:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1060 00000100:00100000:2.0:1767319346.068362:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767319346.068723:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319346.073017:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619712 00000100:00100000:3.0:1767319346.073033:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619712 00000100:00100000:3.0:1767319346.073101:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1394 00000100:00100000:3.0:1767319346.073111:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915619712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319346.073133:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319346.073212:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915619712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 101us (4858us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319346.073219:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1394 00010000:00010000:1.0:1767319346.308566:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319346.628810:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982792320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319346.628905:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319346.629172:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792320 00000100:00100000:3.0:1767319346.629183:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792320 00000100:00080000:2.0:1767319346.629215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319346.629254:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319346.629259:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767319346.629267:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982792448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319346.629288:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982792320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319346.629318:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319346.629325:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319346.629354:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792448 00000100:00100000:0.0:1767319346.629357:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792448 00000100:00100000:3.0:1767319346.629374:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982792320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (467us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319346.629382:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00080000:2.0:1767319346.629441:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319346.629471:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982792448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319346.629486:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319346.629508:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982792448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (184us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319346.629578:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982792448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319346.629587:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982792320:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319346.633298:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767319346.634628:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00010000:00010000:1.0:1767319347.332884:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319348.356628:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319349.189904:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319349.189915:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319349.190005:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319349.190018:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319349.190023:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319349.190031:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4592 00000100:00080000:2.0:1767319349.190035:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319349.190041:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319349.190064:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163982792704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319349.190103:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767319349.190104:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319349.190107:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319349.190109:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319349.190112:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4592 00000100:00100000:0.0:1767319349.190115:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982792832:0@lo:400:kworker.0 00000100:00080000:2.0:1767319349.190115:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319349.190119:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319349.190129:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163982792960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319349.190142:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319349.190145:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319349.190146:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319349.190149:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4592 00000100:00080000:2.0:1767319349.190151:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319349.190154:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319349.190162:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3800 x1853163982793088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319349.190186:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319349.190189:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319349.190191:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319349.190191:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319349.190194:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4592 00000100:00080000:2.0:1767319349.190196:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319349.190199:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319349.190209:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982793472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319349.190234:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982792576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319349.190255:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319349.191730:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982793344:0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.191828:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319349.191881:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319349.191888:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982793216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319349.191910:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319349.191980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982792704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.192146:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319349.193261:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793344 00000100:00100000:3.0:1767319349.193285:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793344 00000100:00100000:3.0:1767319349.193391:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793216 00000100:00100000:3.0:1767319349.193394:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793216 00000100:00100000:3.0:1767319349.193434:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00100000:3.0:1767319349.193444:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982793344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.193550:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982793344:12345-0@lo:400:kworker.0 Request processed in 108us (1721us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319349.193557:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00100000:3.0:1767319349.193574:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00100000:3.0:1767319349.193581:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982793216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.193608:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982793216:12345-0@lo:400:kworker.0 Request processed in 29us (1699us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319349.193614:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00080000:3.0:1767319349.193675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319349.193679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319349.193749:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982793216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319349.193762:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982793344:0@lo:400:kworker.0 00000100:00100000:0.0:1767319349.206655:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982792960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.206730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319349.206909:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792832 00000100:00100000:2.0:1767319349.206918:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792832 00000100:00100000:2.0:1767319349.207009:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792576 00000100:00100000:2.0:1767319349.207013:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792576 00000100:00100000:2.0:1767319349.207032:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792704 00000100:00100000:2.0:1767319349.207035:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792704 00000100:00100000:2.0:1767319349.207048:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061 00000100:00100000:2.0:1767319349.207055:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982792832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319349.207114:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982792832:12345-0@lo:400:kworker.0 Request processed in 58us (16922us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319349.207122:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061 00000100:00100000:2.0:1767319349.207138:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062 00000100:00100000:2.0:1767319349.207144:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982792576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319349.207169:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982792576:12345-0@lo:400:kworker.0 Request processed in 26us (16913us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319349.207175:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062 00000100:00100000:2.0:1767319349.207186:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1395 00000100:00100000:2.0:1767319349.207190:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982792704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319349.207227:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319349.207256:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982792704:12345-0@lo:38:kworker.0 Request processed in 65us (15110us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319349.207262:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1395 00000100:00080000:2.0:1767319349.207302:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319349.207440:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0380 x1853163982792832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319365 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319349.207454:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982792832:0@lo:400:kworker.0 00010000:00080000:2.0:1767319349.207465:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3100 x1853163982792576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319365 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319349.207472:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982792576:0@lo:400:kworker.0 00000100:00100000:0.0:1767319349.208305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982793088:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.208348:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319349.208378:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982793472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.208410:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319349.208448:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319349.208453:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319349.208457:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982792704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319349.208484:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982792960 00000100:00100000:0.0:1767319349.208489:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982792960 00000100:00100000:0.0:1767319349.208563:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793472 00000100:00100000:0.0:1767319349.208567:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793472 00000100:00100000:0.0:1767319349.208582:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1396 00000100:00100000:0.0:1767319349.208588:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982792960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319349.208604:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319349.208639:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982792960:12345-0@lo:38:kworker.0 Request processed in 49us (1908us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319349.208645:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1396 00000100:00100000:0.0:1767319349.208660:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1397 00000100:00100000:0.0:1767319349.208664:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982793472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319349.208672:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319349.208694:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982793472:12345-0@lo:38:kworker.0 Request processed in 29us (284us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319349.208700:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1397 00000100:00080000:0.0:1767319349.208725:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319349.208728:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319349.208732:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982792960:0@lo:38:kworker.0 00000100:00080000:0.0:1767319349.208740:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319349.208743:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319349.208745:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982793472:0@lo:38:kworker.0 00000100:00100000:2.0:1767319349.209435:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793088 00000100:00100000:2.0:1767319349.209443:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793088 00000100:00100000:2.0:1767319349.209480:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 374 00000100:00100000:2.0:1767319349.209487:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982793088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319349.209498:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319349.209544:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982793088:12345-0@lo:38:kworker.0 Request processed in 56us (1195us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319349.209551:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 374 00000100:00080000:0.0:1767319349.209801:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319349.209806:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319349.209810:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982793088:0@lo:38:kworker.0 00010000:00010000:1.0:1767319349.380598:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319350.407067:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319351.185646:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319351.187113:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319351.195717:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620096 00000100:00100000:3.0:1767319351.195733:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620096 00000100:00100000:3.0:1767319351.196162:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915619968 00000100:00100000:3.0:1767319351.196167:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915619968 00000100:00100000:3.0:1767319351.196193:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1063 00000100:00100000:3.0:1767319351.196204:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620096:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319351.196303:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620096:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (10669us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319351.196311:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1063 00000100:00100000:3.0:1767319351.196345:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1398 00000100:00100000:3.0:1767319351.196351:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915619968:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319351.196371:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319351.196395:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915619968:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (9288us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319351.196401:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1398 00000100:00080000:0.0:1767319351.197182:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319351.429347:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319351.748638:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319351.748663:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982793600:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319351.748764:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319351.748982:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793600 00000100:00100000:2.0:1767319351.749003:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793600 00000100:00100000:2.0:1767319351.749082:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 724 00000100:00100000:2.0:1767319351.749093:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982793600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319351.749127:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319351.749172:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982793600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (409us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319351.749180:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 724 00000100:00100000:1.0:1767319351.749298:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982793600:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319351.753141:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319351.753204:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319351.753225:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982793728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319351.753352:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319351.753412:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793728 00000100:00100000:2.0:1767319351.753419:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793728 00000100:00080000:3.0:1767319351.753468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319351.753485:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 725 00000100:00100000:2.0:1767319351.753497:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982793728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319351.753532:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319351.753583:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982793728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319351.753593:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 725 00000100:00100000:3.0:1767319351.753624:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982793728:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319352.452811:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319353.479023:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319354.308864:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319354.308876:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319354.309201:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319354.309206:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319354.309246:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319354.309255:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4597 00000100:00080000:2.0:1767319354.309259:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319354.309303:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319354.309329:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163982793984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319354.309360:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319354.309364:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319354.309366:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319354.309370:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4597 00000100:00080000:2.0:1767319354.309373:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319354.309377:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319354.309390:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0a80 x1853163982794240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319354.309400:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319354.309403:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319354.309405:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319354.309408:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4597 00000100:00080000:2.0:1767319354.309411:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319354.309414:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319354.309424:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3b80 x1853163982794368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319354.309464:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319354.309468:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319354.309471:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319354.309475:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4597 00000100:00080000:2.0:1767319354.309477:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319354.309480:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319354.309491:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982794752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319354.309518:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982794496:0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.309584:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319354.309644:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319354.309650:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982794624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.309671:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319354.309698:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794496 00000100:00100000:2.0:1767319354.309705:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794496 00000100:00100000:2.0:1767319354.309750:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794624 00000100:00100000:2.0:1767319354.309754:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794624 00000100:00100000:2.0:1767319354.309773:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 673 00000100:00100000:2.0:1767319354.309782:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982794496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.309830:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982794496:12345-0@lo:400:kworker.0 Request processed in 49us (246us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319354.309838:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 673 00000100:00100000:2.0:1767319354.309854:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 674 00000100:00100000:2.0:1767319354.309860:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982794624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.309883:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982794624:12345-0@lo:400:kworker.0 Request processed in 24us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319354.309888:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 674 00000100:00080000:2.0:1767319354.309918:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319354.309935:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319354.309974:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982794624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319354.309985:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982794496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319354.310549:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982793856:0@lo:400:kworker.0 00000100:00100000:3.0:1767319354.310591:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319354.310632:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319354.310638:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982794112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319354.310661:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319354.310752:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793856 00000100:00100000:1.0:1767319354.310757:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793856 00000100:00100000:1.0:1767319354.310807:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794112 00000100:00100000:1.0:1767319354.310810:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794112 00000100:00100000:1.0:1767319354.310833:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064 00000100:00100000:1.0:1767319354.310839:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982793856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.310883:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982793856:12345-0@lo:400:kworker.0 Request processed in 43us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319354.310890:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064 00000100:00100000:0.0:1767319354.310930:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982793984:0@lo:38:kworker.0 00000100:00100000:0.0:1767319354.310985:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319354.311007:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982794240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319354.311029:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319354.311044:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982794368:0@lo:38:kworker.0 00000100:00100000:0.0:1767319354.311067:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319354.311089:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982794752:0@lo:38:kworker.0 00000100:00100000:0.0:1767319354.311109:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319354.311131:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982793984 00000100:00100000:0.0:1767319354.311135:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982793984 00000100:00100000:0.0:1767319354.311149:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794240 00000100:00100000:0.0:1767319354.311151:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794240 00000100:00100000:0.0:1767319354.311159:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794752 00000100:00100000:0.0:1767319354.311161:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794752 00000100:00100000:0.0:1767319354.311170:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065 00000100:00100000:0.0:1767319354.311177:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982794112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319354.311208:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982794112:12345-0@lo:400:kworker.0 Request processed in 32us (546us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319354.311209:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1399 00000100:00100000:0.0:1767319354.311214:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065 00000100:00100000:1.0:1767319354.311214:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982793984:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319354.311224:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1400 00000100:00100000:0.0:1767319354.311229:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982794240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319354.311232:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767319354.311236:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319354.311261:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982794240:12345-0@lo:38:kworker.0 Request processed in 31us (231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319354.311267:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1400 00000100:00080000:3.0:1767319354.311344:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319354.315504:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982793984:12345-0@lo:38:kworker.0 Request processed in 4276us (4506us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319354.315566:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1401 00000100:00100000:2.0:1767319354.315576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982794752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319354.315598:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319354.315661:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982794752:12345-0@lo:38:kworker.0 Request processed in 85us (4550us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319354.315667:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1401 00000100:00100000:2.0:1767319354.315780:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794368 00000100:00100000:2.0:1767319354.315788:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794368 00000100:00100000:2.0:1767319354.315823:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 375 00000100:00100000:2.0:1767319354.315828:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982794368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319354.315838:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319354.315867:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982794368:12345-0@lo:38:kworker.0 Request processed in 39us (4800us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319354.315873:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 375 00000100:00080000:2.0:1767319354.316419:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.316426:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.316430:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982793984:0@lo:38:kworker.0 00000100:00080000:2.0:1767319354.316445:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.316460:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.316463:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982794240:0@lo:38:kworker.0 00000100:00080000:2.0:1767319354.316475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.316479:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.316482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982794368:0@lo:38:kworker.0 00000100:00080000:2.0:1767319354.316491:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319354.316494:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319354.316497:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982794752:0@lo:38:kworker.0 00010000:00080000:0.0:1767319354.316939:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1180 x1853163982794112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319370 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319354.316953:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982794112:0@lo:400:kworker.0 00010000:00080000:0.0:1767319354.316965:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f2d80 x1853163982793856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319370 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319354.316973:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982793856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319354.327810:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1399 00010000:00010000:1.0:1767319354.500671:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319355.524570:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319356.301418:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319356.301465:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319356.301542:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620352 00000100:00100000:1.0:1767319356.301553:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620352 00000100:00100000:1.0:1767319356.301624:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620224 00000100:00100000:1.0:1767319356.301628:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620224 00000100:00100000:1.0:1767319356.301645:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1066 00000100:00100000:1.0:1767319356.301655:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915620352:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319356.301745:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915620352:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (337us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319356.301754:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1066 00000100:00100000:1.0:1767319356.301774:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1402 00000100:00100000:1.0:1767319356.301779:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915620224:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319356.301800:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319356.301827:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915620224:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319356.301834:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1402 00000100:00080000:2.0:1767319356.302051:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319356.548560:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319356.868608:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319356.868632:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982794880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319356.868701:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319356.868874:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982794880 00000100:00100000:2.0:1767319356.868884:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982794880 00000100:00080000:3.0:1767319356.868925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319356.868965:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00100000:2.0:1767319356.868976:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982794880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319356.869005:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319356.869059:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982794880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (357us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319356.869066:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00000100:00100000:3.0:1767319356.869578:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982794880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319356.871113:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319356.871135:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982795008:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319356.871205:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319356.874840:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795008 00000100:00100000:3.0:1767319356.874855:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795008 00000100:00100000:3.0:1767319356.875012:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 727 00000100:00100000:3.0:1767319356.875023:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982795008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319356.875057:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319356.875170:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982795008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 148us (3964us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319356.875178:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 727 00000100:00100000:1.0:1767319356.876714:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982795008:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319356.883055:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319357.573313:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319358.596742:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319359.428491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.428502:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319359.428577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.428582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.428585:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.428593:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.428598:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.428604:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319359.428652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e680 x1853163982795264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319359.428689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.428693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.428694:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.428698:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.428706:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.428714:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319359.428727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ca80 x1853163982795520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319359.428737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.428740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.428742:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.428745:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.428748:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.428751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319359.428763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543fb80 x1853163982795648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319359.428794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319359.428798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319359.428800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319359.428803:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4602 00000100:00080000:3.0:1767319359.428805:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319359.428821:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319359.428835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d880 x1853163982796032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319359.428863:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982795904:0@lo:400:kworker.0 00000100:00100000:3.0:1767319359.428928:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.428992:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319359.428999:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982795776:0@lo:400:kworker.0 00000100:00100000:3.0:1767319359.429022:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.429053:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795904 00000100:00100000:3.0:1767319359.429060:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795904 00000100:00100000:1.0:1767319359.429066:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982795392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429108:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.429114:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795776 00000100:00100000:3.0:1767319359.429118:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795776 00000100:00100000:1.0:1767319359.429136:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319359.429145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982795136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319359.429148:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 675 00000100:00100000:3.0:1767319359.429161:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982795904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429186:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319359.429227:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982795904:12345-0@lo:400:kworker.0 Request processed in 69us (299us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319359.429233:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 675 00000100:00100000:1.0:1767319359.429237:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795392 00000100:00100000:1.0:1767319359.429243:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795392 00000100:00100000:3.0:1767319359.429252:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 676 00000100:00100000:3.0:1767319359.429260:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982795776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429313:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067 00000100:00100000:3.0:1767319359.429323:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982795776:12345-0@lo:400:kworker.0 Request processed in 64us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319359.429325:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982795392:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319359.429338:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 676 00000100:00100000:1.0:1767319359.429377:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982795392:12345-0@lo:400:kworker.0 Request processed in 57us (270us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319359.429387:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982795776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429388:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067 00000100:00100000:3.0:1767319359.429406:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982795904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429407:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795136 00000100:00100000:1.0:1767319359.429413:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795136 00000100:00100000:1.0:1767319359.429443:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068 00000100:00100000:1.0:1767319359.429451:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982795136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319359.429480:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982795136:12345-0@lo:400:kworker.0 Request processed in 31us (307us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319359.429487:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068 00000100:00080000:2.0:1767319359.429508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319359.429513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319359.429514:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:1.0:1767319359.429522:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0ab100 x1853163982795136/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319375 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319359.429534:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982795136:0@lo:400:kworker.0 00010000:00080000:1.0:1767319359.429545:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543ce00 x1853163982795392/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319375 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319359.429554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982795392:0@lo:400:kworker.0 00000100:00100000:0.0:1767319359.429597:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982795264:0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429629:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319359.429662:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982795520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429683:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319359.429708:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982795648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429724:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319359.429746:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982796032:0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429794:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319359.429804:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795648 00000100:00100000:1.0:1767319359.429810:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795648 00000100:00100000:0.0:1767319359.429824:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795264 00000100:00100000:0.0:1767319359.429830:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795264 00000100:00100000:1.0:1767319359.429853:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 376 00000100:00100000:0.0:1767319359.429854:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982795520 00000100:00100000:0.0:1767319359.429860:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982795520 00000100:00100000:1.0:1767319359.429861:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982795648:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429876:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796032 00000100:00100000:0.0:1767319359.429880:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796032 00010000:00020000:1.0:1767319359.429882:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319359.429894:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1403 00000100:00100000:0.0:1767319359.429899:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982795264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319359.429911:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319359.429924:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982795648:12345-0@lo:38:kworker.0 Request processed in 62us (196us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319359.429932:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 376 00000100:00100000:0.0:1767319359.429946:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982795264:12345-0@lo:38:kworker.0 Request processed in 44us (315us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319359.429953:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1403 00000100:00100000:1.0:1767319359.429967:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1404 00000100:00100000:0.0:1767319359.429977:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1405 00000100:00100000:1.0:1767319359.429977:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982795520:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319359.429986:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982796032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319359.429992:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767319359.429998:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319359.430029:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982796032:12345-0@lo:38:kworker.0 Request processed in 44us (233us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319359.430035:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1405 00000100:00100000:1.0:1767319359.430044:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982795520:12345-0@lo:38:kworker.0 Request processed in 67us (357us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319359.430053:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1404 00000100:00080000:0.0:1767319359.430062:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319359.430069:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319359.430074:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982795264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319359.430084:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319359.430088:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319359.430091:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982795520:0@lo:38:kworker.0 00000100:00080000:0.0:1767319359.430098:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319359.430101:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319359.430104:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982795648:0@lo:38:kworker.0 00000100:00080000:0.0:1767319359.430111:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319359.430114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319359.430116:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982796032:0@lo:38:kworker.0 00010000:00010000:1.0:1767319359.621092:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319360.645793:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319361.423866:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319361.423962:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319361.424311:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620608 00000100:00100000:3.0:1767319361.424321:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620608 00000100:00100000:3.0:1767319361.424494:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620480 00000100:00100000:3.0:1767319361.424498:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620480 00000100:00100000:3.0:1767319361.424580:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1069 00000100:00100000:3.0:1767319361.424590:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620608:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767319361.424695:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319361.424711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1406 00000100:00100000:1.0:1767319361.424717:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915620480:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319361.424736:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319361.424813:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620608:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 225us (955us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319361.424821:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1069 00000100:00100000:1.0:1767319361.424874:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915620480:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 156us (912us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319361.424881:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1406 00000400:00000100:2.0:1767319361.425046:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915620736 offset 0 length 520: 4 00010000:00010000:1.0:1767319361.669309:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319361.989680:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319361.989706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982796160:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319361.989780:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319361.989904:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796160 00000100:00100000:0.0:1767319361.989913:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796160 00000100:00100000:0.0:1767319361.989974:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 728 00000100:00100000:0.0:1767319361.989984:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982796160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319361.990025:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319361.990069:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982796160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319361.990077:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 728 00000100:00080000:1.0:1767319361.990261:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319361.990318:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982796160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319361.994374:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319361.994402:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982796288:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319361.994476:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319361.994617:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796288 00000100:00100000:0.0:1767319361.994626:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796288 00000100:00100000:0.0:1767319361.994726:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 729 00000100:00100000:0.0:1767319361.994737:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982796288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319361.994772:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319361.994814:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982796288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319361.994820:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 729 00000100:00080000:2.0:1767319361.994901:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319361.994973:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982796288:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319362.692536:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319363.716479:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319364.548485:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319339/real 1767319339] req@ffff9346395e9500 x1853163982789888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319364 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319364.548568:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346395e9500 x1853163982789888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319364 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319364.548594:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319364.548615:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319364.548619:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982789888:0@lo:250:kworker.0 00000100:00080000:3.0:1767319364.548743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319364.548764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319364.548767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319364.548772:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4587 00000100:00080000:3.0:1767319364.548776:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319364.548783:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319364.548816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163982796416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319364.548865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319364.548868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319364.548870:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319364.548874:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4607 00000100:00080000:3.0:1767319364.548876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319364.548880:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319364.548906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c700 x1853163982796672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319364.548931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319364.548934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319364.548936:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319364.548939:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4607 00000100:00080000:3.0:1767319364.548941:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319364.548957:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319364.548972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f100 x1853163982796928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319364.548981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319364.548984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319364.548994:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319364.549000:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4607 00000100:00080000:3.0:1767319364.549002:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319364.549009:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319364.549018:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543df80 x1853163982797056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319364.549048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319364.549051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319364.549053:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319364.549056:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4607 00000100:00080000:3.0:1767319364.549059:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319364.549062:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319364.549071:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f800 x1853163982797440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319364.549109:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982797184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.549176:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.549228:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319364.549232:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982797312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.549246:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.549268:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797184 00000100:00100000:3.0:1767319364.549290:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797184 00000100:00100000:3.0:1767319364.549325:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797312 00000100:00100000:3.0:1767319364.549327:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797312 00000100:00100000:3.0:1767319364.549339:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 677 00000100:00100000:3.0:1767319364.549344:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982797184:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549375:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982796544:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.549392:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982797184:12345-0@lo:400:kworker.0 Request processed in 45us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319364.549400:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 677 00000100:00100000:3.0:1767319364.549419:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 678 00000100:00100000:3.0:1767319364.549425:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982797312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549442:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.549462:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982797312:12345-0@lo:400:kworker.0 Request processed in 36us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319364.549469:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 678 00000100:00100000:1.0:1767319364.549480:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319364.549488:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982796800:0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549517:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319364.549521:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982797312:0@lo:400:kworker.0 00000100:00100000:3.0:1767319364.549538:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982797184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549556:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796544 00000100:00100000:1.0:1767319364.549561:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796544 00000100:00100000:1.0:1767319364.549619:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796800 00000100:00100000:1.0:1767319364.549622:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796800 00000100:00100000:1.0:1767319364.549645:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070 00000100:00100000:1.0:1767319364.549652:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982796544:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549699:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982796544:12345-0@lo:400:kworker.0 Request processed in 47us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319364.549708:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070 00000100:00080000:2.0:1767319364.549718:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319364.549724:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319364.549725:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071 00000100:00080000:2.0:1767319364.549727:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319364.549734:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982796800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319364.549741:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982796416:0@lo:250:kworker.0 00000100:00100000:1.0:1767319364.549765:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982796800:12345-0@lo:400:kworker.0 Request processed in 33us (247us total) trans 0 rc 0/0 00000400:00000100:2.0:1767319364.549773:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982796416 offset 0 length 520: 4 00000100:00100000:1.0:1767319364.549774:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071 00000100:00100000:2.0:1767319364.549805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982796672:0@lo:38:kworker.0 00010000:00080000:1.0:1767319364.549815:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543ea00 x1853163982796800/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319380 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319364.549827:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982796800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319364.549834:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767319364.549844:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543d500 x1853163982796544/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319380 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319364.549855:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982796928:0@lo:38:kworker.0 00000100:00100000:1.0:1767319364.549856:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982796544:0@lo:400:kworker.0 00000100:00100000:1.0:1767319364.549884:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796672 00000100:00100000:2.0:1767319364.549885:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319364.549890:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796672 00000100:00100000:2.0:1767319364.549903:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982797056:0@lo:38:kworker.0 00000100:00100000:1.0:1767319364.549923:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982796928 00000100:00100000:1.0:1767319364.549927:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982796928 00000100:00100000:1.0:1767319364.549943:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1407 00000100:00100000:2.0:1767319364.549949:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319364.549954:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982796672:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.549971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982797440:0@lo:38:kworker.0 00010000:00020000:1.0:1767319364.549972:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319364.550011:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982796672:12345-0@lo:38:kworker.0 Request processed in 59us (180us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319364.550021:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1407 00000100:00100000:2.0:1767319364.550021:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319364.550037:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797440 00000100:00100000:1.0:1767319364.550040:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797440 00000100:00080000:2.0:1767319364.550051:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319364.550054:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1408 00000100:00080000:2.0:1767319364.550058:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319364.550063:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982796928:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.550066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982796672:0@lo:38:kworker.0 00010000:00020000:1.0:1767319364.550077:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.550154:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797056 00000100:00100000:2.0:1767319364.550166:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797056 00000100:00100000:3.0:1767319364.550172:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1409 00000100:00100000:3.0:1767319364.550178:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982797440:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319364.550197:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 377 00010000:00020000:3.0:1767319364.550198:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.550202:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982797056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319364.550211:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319364.550633:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982797056:12345-0@lo:38:kworker.0 Request processed in 428us (686us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319364.550634:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982797440:12345-0@lo:38:kworker.0 Request processed in 455us (616us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319364.550643:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 377 00000100:00100000:3.0:1767319364.550643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1409 00000100:00100000:1.0:1767319364.550649:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982796928:12345-0@lo:38:kworker.0 Request processed in 586us (763us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319364.550658:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1408 00000100:00080000:0.0:1767319364.550659:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319364.550668:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319364.550673:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982796928:0@lo:38:kworker.0 00000100:00080000:0.0:1767319364.550683:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319364.550687:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319364.550690:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982797056:0@lo:38:kworker.0 00000100:00080000:0.0:1767319364.550711:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319364.550714:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319364.550717:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982797440:0@lo:38:kworker.0 00010000:00010000:1.0:1767319364.740882:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319365.766185:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319366.540947:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319366.541033:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620864 00000100:00100000:1.0:1767319366.541054:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620864 00000100:00100000:1.0:1767319366.541107:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1410 00000100:00100000:1.0:1767319366.541116:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915620864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319366.541139:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319366.541219:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915620864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (282us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319366.541227:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1410 00000100:00100000:3.0:1767319366.550256:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319366.550334:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915620992 00000100:00100000:2.0:1767319366.550346:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915620992 00000100:00080000:0.0:1767319366.550520:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319366.550739:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1072 00000100:00100000:2.0:1767319366.550751:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319366.550850:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915620992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319366.550859:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1072 00010000:00010000:1.0:1767319366.789951:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319367.116360:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319367.116386:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982797568:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319367.116456:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319367.116663:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797568 00000100:00100000:0.0:1767319367.116672:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797568 00000100:00100000:3.0:1767319367.116673:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982797696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319367.116699:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319367.116764:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319367.116791:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797696 00000100:00100000:1.0:1767319367.116796:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797696 00000100:00100000:1.0:1767319367.116857:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 730 00000100:00100000:1.0:1767319367.116867:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982797696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319367.116898:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319367.116971:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00100000:0.0:1767319367.116979:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982797568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319367.116998:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319367.117165:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982797696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 299us (463us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319367.117173:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 730 00000100:00100000:3.0:1767319367.117180:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982797696:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319367.117205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319367.120858:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982797568:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319367.120880:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982797568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3829us (4350us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319367.120894:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00010000:00010000:1.0:1767319367.814265:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319368.836468:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319369.670240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319369.670250:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319369.670332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319369.670336:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319369.670339:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319369.670346:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4613 00000100:00080000:2.0:1767319369.670350:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319369.670357:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319369.670380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982797952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319369.670407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319369.670410:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319369.670412:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319369.670415:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4613 00000100:00080000:2.0:1767319369.670418:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319369.670420:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319369.670430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9500 x1853163982798208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319369.670437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319369.670440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319369.670441:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319369.670445:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4613 00000100:00080000:2.0:1767319369.670446:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319369.670449:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319369.670499:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4e00 x1853163982798336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319369.670525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319369.670528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319369.670529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319369.670532:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4613 00000100:00080000:2.0:1767319369.670534:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319369.670537:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319369.670544:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5180 x1853163982798720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319369.670589:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982798464:0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.670598:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982798080:0@lo:400:kworker.0 00000100:00100000:0.0:1767319369.670727:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982797952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319369.670789:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319369.670820:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319369.670826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982797824:0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.670845:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319369.670871:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798080 00000100:00100000:2.0:1767319369.670877:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798080 00000100:00100000:2.0:1767319369.670927:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797824 00000100:00100000:2.0:1767319369.670930:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797824 00000100:00100000:2.0:1767319369.670950:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073 00000100:00100000:2.0:1767319369.670957:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982798080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.671011:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982798080:12345-0@lo:400:kworker.0 Request processed in 54us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319369.671018:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073 00000100:00100000:2.0:1767319369.671035:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074 00000100:00100000:2.0:1767319369.671043:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982797824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.671071:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982797824:12345-0@lo:400:kworker.0 Request processed in 30us (224us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319369.671076:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074 00010000:00080000:2.0:1767319369.671105:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1180 x1853163982797824/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319385 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319369.671115:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982797824:0@lo:400:kworker.0 00010000:00080000:2.0:1767319369.671124:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f2d80 x1853163982798080/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319385 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319369.671131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982798080:0@lo:400:kworker.0 00000100:00100000:3.0:1767319369.673339:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982798592:0@lo:400:kworker.0 00000100:00100000:3.0:1767319369.673449:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319369.673510:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798592 00000100:00100000:3.0:1767319369.673516:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798592 00000100:00100000:3.0:1767319369.673563:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 679 00000100:00100000:3.0:1767319369.673572:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982798592:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.676302:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982798592:0@lo:400:kworker.0 00000100:00080000:2.0:1767319369.676345:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319369.676348:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319369.686365:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319369.686571:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798464 00000100:00100000:2.0:1767319369.686581:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798464 00000100:00100000:2.0:1767319369.686651:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 680 00000100:00100000:2.0:1767319369.686661:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982798464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319369.686724:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982798464:12345-0@lo:400:kworker.0 Request processed in 65us (366us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319369.686731:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 680 00000100:00080000:1.0:1767319369.687017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319369.687053:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982798464:0@lo:400:kworker.0 00000100:00100000:0.0:1767319369.687144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319369.687180:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982798208:0@lo:38:kworker.0 00000100:00100000:0.0:1767319369.687209:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319369.687234:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982798336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319369.687258:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319369.687291:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982798720:0@lo:38:kworker.0 00000100:00100000:0.0:1767319369.687315:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319369.687432:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982798592:12345-0@lo:400:kworker.0 Request processed in 13858us (13979us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319369.687441:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 679 00000100:00100000:3.0:1767319369.687477:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982797952 00000100:00100000:3.0:1767319369.687482:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982797952 00000100:00100000:3.0:1767319369.687502:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798208 00000100:00100000:3.0:1767319369.687506:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798208 00000100:00100000:3.0:1767319369.687518:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798720 00000100:00100000:3.0:1767319369.687520:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798720 00000100:00100000:3.0:1767319369.687530:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1411 00000100:00100000:3.0:1767319369.687535:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982797952:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319369.688320:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1412 00000100:00100000:1.0:1767319369.688326:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982798208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319369.688338:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319369.688416:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982798208:12345-0@lo:38:kworker.0 Request processed in 89us (1205us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319369.688423:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1412 00000100:00100000:1.0:1767319369.688439:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1413 00000100:00100000:1.0:1767319369.688443:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982798720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319369.688453:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319369.688477:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982798720:12345-0@lo:38:kworker.0 Request processed in 33us (1161us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319369.688483:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1413 00000100:00080000:1.0:1767319369.688533:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.688539:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.688542:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982798208:0@lo:38:kworker.0 00000100:00080000:1.0:1767319369.688551:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.688554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.688557:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982798720:0@lo:38:kworker.0 00010000:00020000:3.0:1767319369.690438:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319369.691034:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982797952:12345-0@lo:38:kworker.0 Request processed in 3493us (3885us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319369.691047:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1411 00000100:00080000:1.0:1767319369.692384:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319369.692393:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319369.692398:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982797952:0@lo:38:kworker.0 00000100:00100000:0.0:1767319369.693100:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798336 00000100:00100000:0.0:1767319369.693111:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798336 00000100:00100000:0.0:1767319369.693208:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 378 00000100:00100000:0.0:1767319369.693216:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982798336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319369.693242:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319369.693342:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982798336:12345-0@lo:38:kworker.0 Request processed in 125us (6082us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319369.693349:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 378 00000100:00080000:2.0:1767319369.693425:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319369.693429:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319369.693433:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982798336:0@lo:38:kworker.0 00010000:00010000:1.0:1767319369.871852:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319370.885457:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319371.660356:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319371.660443:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319371.664874:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621120 00000100:00100000:0.0:1767319371.666763:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621120 00000100:00100000:1.0:1767319371.666968:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621248 00000100:00100000:1.0:1767319371.666974:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621248 00000100:00100000:1.0:1767319371.667034:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1075 00000100:00100000:1.0:1767319371.667044:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915621248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319371.667124:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915621248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (6681us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319371.667131:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1075 00000100:00100000:1.0:1767319371.667150:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1414 00000100:00100000:1.0:1767319371.667155:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d4c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915621120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319371.667174:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319371.667198:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d4c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915621120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (6851us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319371.667204:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1414 00000100:00080000:0.0:1767319371.669117:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319371.909682:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319372.228592:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982798848:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319372.228669:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319372.228716:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319372.228722:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982798976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319372.228741:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798848 00000100:00100000:0.0:1767319372.228744:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319372.228749:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798848 00000100:00080000:0.0:1767319372.228776:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319372.228800:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982798976 00000100:00100000:1.0:1767319372.228803:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982798976 00000100:00100000:0.0:1767319372.228810:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000100:00100000:0.0:1767319372.228819:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982798848:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319372.228821:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 733 00000100:00100000:1.0:1767319372.228826:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982798976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319372.228847:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:1.0:1767319372.228847:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319372.228885:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982798848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (216us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319372.228892:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00100000:1.0:1767319372.228892:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982798976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (147us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319372.228897:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 733 00000100:00080000:0.0:1767319372.228910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319372.228953:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982798976:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319372.228967:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982798848:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319372.932918:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319373.958201:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319374.788755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319374.788767:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319374.788866:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:3.0:1767319374.789113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319374.789118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319374.789121:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319374.789129:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4618 00000100:00080000:3.0:1767319374.789133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319374.789140:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319374.789165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e680 x1853163982799232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319374.789232:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982799232:0@lo:38:kworker.0 00000100:00100000:1.0:1767319374.789313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319374.791396:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799232 00000100:00100000:0.0:1767319374.791409:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799232 00000100:00100000:0.0:1767319374.791469:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1415 00000100:00100000:0.0:1767319374.791477:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982799232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319374.791500:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319374.791577:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982799232:12345-0@lo:38:kworker.0 Request processed in 99us (2264us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319374.791585:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1415 00000100:00080000:0.0:1767319374.791629:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319374.791635:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319374.791639:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982799232:0@lo:38:kworker.0 00000100:00100000:2.0:1767319374.794162:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982799360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319374.794234:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319374.794360:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982799104:0@lo:400:kworker.0 00000100:00100000:2.0:1767319374.794382:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319374.794807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319374.794813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319374.794816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319374.794821:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4618 00000100:00080000:3.0:1767319374.794826:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319374.794842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319374.794873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f800 x1853163982799488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319374.794904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319374.794908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319374.794910:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319374.794913:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4618 00000100:00080000:3.0:1767319374.794915:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319374.794919:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319374.794931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543df80 x1853163982799616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319374.794985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319374.794988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319374.794990:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319374.794994:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4618 00000100:00080000:3.0:1767319374.794998:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319374.795000:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319374.795010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982800000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319374.795042:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982799872:0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.795077:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319374.795103:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319374.795109:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982799744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.795131:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319374.795159:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982799488:0@lo:38:kworker.0 00000100:00100000:3.0:1767319374.795189:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799872 00000100:00100000:3.0:1767319374.795196:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799872 00000100:00100000:3.0:1767319374.795247:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799744 00000100:00100000:3.0:1767319374.795251:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799744 00000100:00100000:3.0:1767319374.795288:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 681 00000100:00100000:3.0:1767319374.795297:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982799872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.795345:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982799872:12345-0@lo:400:kworker.0 Request processed in 49us (266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319374.795352:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 681 00000100:00100000:3.0:1767319374.795367:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 682 00000100:00100000:3.0:1767319374.795373:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982799744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.795411:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982799744:12345-0@lo:400:kworker.0 Request processed in 38us (280us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319374.795416:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 682 00000100:00100000:3.0:1767319374.795450:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982799744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.795460:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982799872:0@lo:400:kworker.0 00000100:00080000:0.0:1767319374.795878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319374.795882:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319374.799457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319374.799567:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799360 00000100:00100000:1.0:1767319374.799578:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799360 00000100:00100000:1.0:1767319374.799667:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799104 00000100:00100000:1.0:1767319374.799671:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799104 00000100:00100000:1.0:1767319374.799698:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076 00000100:00100000:1.0:1767319374.799707:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982799360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319374.799788:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982799360:12345-0@lo:400:kworker.0 Request processed in 82us (5553us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319374.799795:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076 00000100:00100000:1.0:1767319374.799813:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077 00000100:00100000:1.0:1767319374.799820:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982799104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319374.799848:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982799104:12345-0@lo:400:kworker.0 Request processed in 29us (5465us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319374.799854:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077 00000100:00080000:2.0:1767319374.800516:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319374.800556:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543c380 x1853163982799360/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319390 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319374.800573:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982799360:0@lo:400:kworker.0 00010000:00080000:2.0:1767319374.800587:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543ce00 x1853163982799104/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319390 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319374.800596:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982799104:0@lo:400:kworker.0 00000100:00100000:3.0:1767319374.804302:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982799616:0@lo:38:kworker.0 00000100:00100000:3.0:1767319374.804366:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319374.804405:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982800000:0@lo:38:kworker.0 00000100:00100000:3.0:1767319374.804431:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319374.806191:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799616 00000100:00100000:2.0:1767319374.806205:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799616 00000100:00100000:2.0:1767319374.806255:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 379 00000100:00100000:2.0:1767319374.806262:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982799616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319374.806305:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319374.806362:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982799616:12345-0@lo:38:kworker.0 Request processed in 99us (1995us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319374.806368:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 379 00000100:00080000:3.0:1767319374.811692:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319374.811705:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319374.811710:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982799616:0@lo:38:kworker.0 00000100:00100000:0.0:1767319374.812213:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982799488 00000100:00100000:0.0:1767319374.812222:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982799488 00000100:00100000:0.0:1767319374.812326:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800000 00000100:00100000:0.0:1767319374.812331:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800000 00000100:00100000:2.0:1767319374.812378:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1416 00000100:00100000:2.0:1767319374.812397:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982799488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319374.812421:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319374.812496:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982799488:12345-0@lo:38:kworker.0 Request processed in 108us (13046us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319374.812504:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1416 00000100:00100000:2.0:1767319374.812522:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1417 00000100:00100000:2.0:1767319374.812527:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982800000:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319374.812537:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319374.812561:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982800000:12345-0@lo:38:kworker.0 Request processed in 32us (8129us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319374.812566:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1417 00000100:00080000:3.0:1767319374.812711:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319374.812717:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319374.812720:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982799488:0@lo:38:kworker.0 00000100:00080000:3.0:1767319374.812730:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319374.812733:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319374.812736:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982800000:0@lo:38:kworker.0 00010000:00010000:1.0:1767319374.980765:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319376.004684:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319376.781075:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319376.783818:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621504 00000100:00100000:1.0:1767319376.783834:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621504 00000100:00100000:1.0:1767319376.783932:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1078 00000100:00100000:1.0:1767319376.783942:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915621504:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319376.784017:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915621504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (2953us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319376.784025:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1078 00000100:00100000:2.0:1767319376.784148:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319376.784312:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621376 00000100:00100000:3.0:1767319376.784318:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621376 00000100:00100000:3.0:1767319376.784353:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1418 00000100:00100000:3.0:1767319376.784359:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915621376:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319376.784378:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319376.784415:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915621376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (268us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319376.784421:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1418 00000100:00080000:0.0:1767319376.784652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319377.028600:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319377.350194:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319377.351821:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982800256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319377.352129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319377.352255:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982800128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319377.352297:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319377.356131:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800256 00000100:00100000:3.0:1767319377.356146:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800256 00000100:00100000:3.0:1767319377.356214:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800128 00000100:00100000:3.0:1767319377.356217:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800128 00000100:00100000:3.0:1767319377.356235:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00100000:3.0:1767319377.356244:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982800256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319377.356287:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319377.356354:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982800256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (4225us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319377.356362:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00000100:00100000:3.0:1767319377.356380:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00100000:3.0:1767319377.356387:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982800128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319377.356403:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319377.356428:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982800128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (4131us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319377.356434:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00100000:3.0:1767319377.356478:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982800256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319377.356492:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982800128:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319377.356571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319377.356576:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319378.053106:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319379.077339:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319379.908460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319379.908471:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319379.908541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319379.908545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319379.908550:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319379.908557:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4623 00000100:00080000:2.0:1767319379.908562:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319379.908569:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319379.908593:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7100 x1853163982800512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319379.908632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319379.908636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319379.908638:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319379.908642:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4623 00000100:00080000:2.0:1767319379.908644:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319379.908648:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319379.908659:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4380 x1853163982800768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319379.908668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319379.908670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319379.908672:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319379.908675:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4623 00000100:00100000:1.0:1767319379.908676:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982800512:0@lo:38:kworker.0 00000100:00080000:2.0:1767319379.908677:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319379.908681:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319379.908690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163982800896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319379.908722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319379.908725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319379.908727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319379.908730:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4623 00000100:00080000:2.0:1767319379.908732:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319379.908735:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319379.908743:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163982801280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319379.908744:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319379.908784:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800512 00000100:00100000:2.0:1767319379.908793:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800512 00000100:00100000:1.0:1767319379.908817:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982800768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319379.908840:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319379.908844:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1419 00000100:00100000:2.0:1767319379.908851:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982800512:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319379.908861:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982800896:0@lo:38:kworker.0 00010000:00020000:2.0:1767319379.908872:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319379.908886:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319379.908909:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982801280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319379.908913:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982800512:12345-0@lo:38:kworker.0 Request processed in 61us (169us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319379.908920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1419 00000100:00100000:1.0:1767319379.908927:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319379.908958:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319379.908963:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319379.908966:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982800512:0@lo:38:kworker.0 00000100:00100000:2.0:1767319379.909015:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800768 00000100:00100000:2.0:1767319379.909020:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800768 00000100:00100000:2.0:1767319379.909036:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801280 00000100:00100000:2.0:1767319379.909038:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801280 00000100:00100000:2.0:1767319379.909048:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1420 00000100:00100000:2.0:1767319379.909052:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982800768:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319379.909063:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319379.909118:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982800384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909145:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319379.909160:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319379.909165:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982800640:0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909183:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319379.909202:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982801024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909267:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319379.909311:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319379.909317:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982801152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909337:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319379.909378:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800384 00000100:00100000:0.0:1767319379.909381:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800384 00000100:00100000:0.0:1767319379.909415:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800640 00000100:00100000:0.0:1767319379.909418:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800640 00000100:00100000:0.0:1767319379.909433:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079 00000100:00100000:0.0:1767319379.909439:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982800384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909480:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982800384:12345-0@lo:400:kworker.0 Request processed in 40us (333us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319379.909515:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079 00000100:00100000:0.0:1767319379.909528:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080 00000100:00100000:0.0:1767319379.909534:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982800640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.909560:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982800640:12345-0@lo:400:kworker.0 Request processed in 27us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319379.909566:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080 00000100:00100000:0.0:1767319379.909577:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1421 00000100:00100000:0.0:1767319379.909581:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982801280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319379.909591:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319379.909645:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982800768:12345-0@lo:38:kworker.0 Request processed in 591us (803us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319379.909655:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1420 00000100:00080000:2.0:1767319379.909671:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319379.910867:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982801280:12345-0@lo:38:kworker.0 Request processed in 1281us (1935us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319379.910876:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1421 00000100:00100000:0.0:1767319379.910902:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982800896 00000100:00100000:0.0:1767319379.910907:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982800896 00000100:00100000:0.0:1767319379.910936:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 380 00000100:00100000:0.0:1767319379.910941:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982800896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319379.910953:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319379.910986:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982800896:12345-0@lo:38:kworker.0 Request processed in 44us (2098us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319379.910992:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 380 00010000:00080000:0.0:1767319379.911026:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5f80 x1853163982800640/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319395 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319379.911038:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982800640:0@lo:400:kworker.0 00010000:00080000:0.0:1767319379.911048:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6a00 x1853163982800384/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319395 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319379.911082:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982800384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.911136:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801024 00000100:00100000:0.0:1767319379.911140:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801024 00000100:00100000:0.0:1767319379.911167:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801152 00000100:00100000:0.0:1767319379.911170:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801152 00000100:00100000:0.0:1767319379.911401:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00100000:0.0:1767319379.911408:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982801024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.911447:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982801024:12345-0@lo:400:kworker.0 Request processed in 39us (2178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319379.911454:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00000100:00100000:0.0:1767319379.911465:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 684 00000100:00100000:0.0:1767319379.911471:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982801152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319379.911496:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982801152:12345-0@lo:400:kworker.0 Request processed in 26us (2159us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319379.911502:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 684 00000100:00100000:1.0:1767319379.911576:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982801152:0@lo:400:kworker.0 00000100:00100000:1.0:1767319379.911605:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982801024:0@lo:400:kworker.0 00000100:00080000:1.0:1767319379.911637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319379.911640:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319379.912296:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319379.912302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319379.912305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982801280:0@lo:38:kworker.0 00000100:00080000:2.0:1767319379.912316:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319379.912319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319379.912322:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982800768:0@lo:38:kworker.0 00000100:00080000:2.0:1767319379.912339:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319379.912342:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319379.912345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982800896:0@lo:38:kworker.0 00010000:00010000:1.0:1767319380.100887:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319381.127112:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319381.907332:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319381.907372:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319381.907513:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621760 00000100:00100000:3.0:1767319381.907523:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621760 00000100:00100000:3.0:1767319381.907600:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915621632 00000100:00100000:3.0:1767319381.907604:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915621632 00000100:00100000:3.0:1767319381.907625:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1081 00000100:00100000:3.0:1767319381.907636:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915621760:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319381.907715:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915621760:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (392us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319381.907723:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1081 00000100:00100000:3.0:1767319381.907742:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1422 00000100:00100000:3.0:1767319381.907748:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915621632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319381.907780:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319381.907809:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915621632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (437us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319381.907815:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1422 00000100:00080000:2.0:1767319381.907980:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319382.149824:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319382.470238:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982801536:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319382.470333:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319382.470412:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319382.470419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982801408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319382.470437:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801536 00000100:00100000:1.0:1767319382.470441:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319382.470456:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801536 00000100:00100000:1.0:1767319382.470471:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801408 00000100:00100000:1.0:1767319382.470475:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801408 00000100:00080000:3.0:1767319382.470571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319382.470575:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319382.470632:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 736 00000100:00100000:1.0:1767319382.470642:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982801408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319382.470685:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319382.470728:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982801408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319382.470738:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 736 00000100:00100000:0.0:1767319382.470932:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982801408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319382.475854:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 737 00000100:00100000:2.0:1767319382.475878:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982801536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319382.475913:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319382.476002:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982801536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 135us (5666us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319382.476010:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 737 00000100:00100000:0.0:1767319382.479434:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982801536:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319383.175211:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319384.196880:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319385.028923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.028933:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319385.029008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.029020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.029025:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.029034:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.029038:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.029045:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319385.029074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7800 x1853163982801792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.029105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319385.029109:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982801664:0@lo:400:kworker.0 00000100:00080000:2.0:1767319385.029110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.029114:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.029120:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.029122:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.029126:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319385.029128:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982801792:0@lo:38:kworker.0 00000100:00100000:2.0:1767319385.029135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6a00 x1853163982802048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.029144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.029146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.029148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.029151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.029154:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.029157:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319385.029167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5f80 x1853163982802176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319385.029200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319385.029204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319385.029207:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319385.029210:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4628 00000100:00080000:2.0:1767319385.029213:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319385.029217:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319385.029223:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982801920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.029230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7b80 x1853163982802560/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319385.029287:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982802304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.030881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.031369:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982802432:0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.031419:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.031456:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801664 00000100:00100000:0.0:1767319385.031464:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801664 00000100:00100000:1.0:1767319385.031794:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319385.031828:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982802048:0@lo:38:kworker.0 00000100:00100000:1.0:1767319385.031860:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.031894:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801792 00000100:00100000:0.0:1767319385.031900:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801792 00000100:00100000:1.0:1767319385.031901:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982802176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319385.031924:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802048 00000100:00100000:1.0:1767319385.031930:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.031944:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802048 00000100:00100000:1.0:1767319385.031959:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982802560:0@lo:38:kworker.0 00000100:00100000:0.0:1767319385.031966:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082 00000100:00100000:1.0:1767319385.031986:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319385.031992:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982801664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319385.032014:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802560 00000100:00100000:1.0:1767319385.032020:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802560 00000100:00100000:1.0:1767319385.032040:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1423 00000100:00100000:1.0:1767319385.032046:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982801792:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319385.032058:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982801664:12345-0@lo:400:kworker.0 Request processed in 72us (1182us total) trans 0 rc 0/0 00010000:00020000:1.0:1767319385.032067:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319385.032068:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082 00000100:00100000:0.0:1767319385.032091:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1424 00000100:00100000:0.0:1767319385.032097:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982802048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319385.032111:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319385.032111:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982801792:12345-0@lo:38:kworker.0 Request processed in 62us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319385.032121:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1423 00000100:00100000:1.0:1767319385.032134:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1425 00000100:00100000:1.0:1767319385.032138:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982802560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319385.032146:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319385.032169:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982802560:12345-0@lo:38:kworker.0 Request processed in 29us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319385.032177:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1425 00000100:00100000:1.0:1767319385.032199:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802176 00000100:00100000:1.0:1767319385.032202:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802176 00000100:00100000:1.0:1767319385.032218:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 381 00000100:00100000:1.0:1767319385.032223:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982802176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319385.032232:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319385.032255:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982802176:12345-0@lo:38:kworker.0 Request processed in 31us (325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319385.032261:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 381 00000100:00080000:1.0:1767319385.032308:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319385.032312:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319385.032316:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982802176:0@lo:38:kworker.0 00000100:00080000:1.0:1767319385.032325:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319385.032328:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319385.032330:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982802560:0@lo:38:kworker.0 00000100:00080000:1.0:1767319385.032842:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319385.032847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319385.032850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982801792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319385.032901:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802432 00000100:00100000:1.0:1767319385.032906:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802432 00000100:00100000:1.0:1767319385.032940:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 685 00000100:00100000:1.0:1767319385.032947:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982802432:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319385.041446:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982802048:12345-0@lo:38:kworker.0 Request processed in 9338us (9576us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319385.041463:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1424 00010000:00080000:0.0:1767319385.043635:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6680 x1853163982801664/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319401 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319385.043656:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982801664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.043687:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319385.043725:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319385.043733:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319385.043737:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982802048:0@lo:38:kworker.0 00000100:00080000:2.0:1767319385.043774:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319385.043781:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319385.043804:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802304 00000100:00100000:2.0:1767319385.043811:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802304 00000100:00100000:3.0:1767319385.043853:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319385.043859:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00100000:2.0:1767319385.043870:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982802304:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319385.043900:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982801920 00000100:00100000:3.0:1767319385.043905:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982801920 00000100:00100000:2.0:1767319385.043933:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982802304:12345-0@lo:400:kworker.0 Request processed in 65us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319385.043942:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00080000:2.0:1767319385.043958:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319385.043961:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319385.043980:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982802304:0@lo:400:kworker.0 00000100:00100000:3.0:1767319385.044723:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1767319385.044733:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982801920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319385.045397:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982801920:12345-0@lo:400:kworker.0 Request processed in 663us (1542us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319385.045406:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083 00010000:00080000:0.0:1767319385.045722:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5c00 x1853163982801920/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319401 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319385.045736:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982801920:0@lo:400:kworker.0 00000100:00100000:2.0:1767319385.045811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982802432:0@lo:400:kworker.0 00000100:00100000:1.0:1767319385.046469:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982802432:12345-0@lo:400:kworker.0 Request processed in 11546us (13073us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319385.046486:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 685 00010000:00010000:1.0:1767319385.221099:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319385.997555:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319385.997771:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622144 00000100:00100000:3.0:1767319385.997783:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622144 00000100:00100000:3.0:1767319385.997871:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1084 00000100:00100000:3.0:1767319385.997882:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915622144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319385.997975:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915622144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319385.997983:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1084 00000100:00080000:2.0:1767319385.998241:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767319385.998774:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915621888 offset 0 length 520: 4 00000100:00100000:3.0:1767319385.998805:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319385.999074:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622016 00000100:00100000:0.0:1767319385.999081:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622016 00000100:00100000:0.0:1767319385.999119:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1426 00000100:00100000:0.0:1767319385.999125:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915622016:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319385.999146:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319385.999199:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915622016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 72us (392us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319385.999206:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1426 00010000:00010000:1.0:1767319386.244632:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319387.270107:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319387.588615:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982802688:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319387.588717:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319387.588814:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802688 00000100:00100000:3.0:1767319387.588822:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802688 00000100:00100000:3.0:1767319387.588879:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00100000:3.0:1767319387.588889:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982802688:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319387.590018:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767319387.595312:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319387.595414:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982802688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 6523us (6694us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319387.595425:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00000100:00100000:0.0:1767319387.595722:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982802688:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319387.595857:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319387.595868:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982802816:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319387.595916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319387.596355:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982802816 00000100:00100000:3.0:1767319387.596364:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982802816 00000100:00100000:3.0:1767319387.596419:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 739 00000100:00100000:3.0:1767319387.596428:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982802816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319387.596447:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319387.596494:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982802816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (578us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319387.596501:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 739 00000100:00100000:3.0:1767319387.596535:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982802816:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319387.596622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319388.293142:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319389.124951:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319364/real 1767319364] req@ffff93460543d180 x1853163982796416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319389 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319389.124988:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460543d180 x1853163982796416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319389 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319389.125013:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319389.125040:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319389.125050:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982796416:0@lo:250:kworker.0 00000100:00080000:0.0:1767319389.125254:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319389.125260:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319389.125263:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319389.125268:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4613 00000100:00080000:0.0:1767319389.125288:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319389.125294:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319389.125326:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52680 x1853163982802944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319389.125387:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319389.125391:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319389.125393:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319389.125397:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4633 00000100:00080000:0.0:1767319389.125399:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319389.125402:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319389.125417:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50380 x1853163982803200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319389.125434:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319389.125437:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319389.125439:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319389.125442:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4633 00000100:00080000:0.0:1767319389.125445:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319389.125448:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319389.125457:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50a80 x1853163982803456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319389.125465:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319389.125467:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319389.125469:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319389.125472:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4633 00000100:00080000:0.0:1767319389.125475:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319389.125478:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319389.125493:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163982803584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319389.125521:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319389.125525:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319389.125527:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319389.125530:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4633 00000100:00080000:0.0:1767319389.125532:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319389.125535:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319389.125545:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982803968/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319389.125571:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982803840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319389.125646:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.125703:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319389.125709:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982803712:0@lo:400:kworker.0 00000100:00100000:0.0:1767319389.125730:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.125800:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982802944:0@lo:250:kworker.0 00000400:00000100:1.0:1767319389.125823:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982802944 offset 0 length 520: 4 00000100:00100000:1.0:1767319389.125841:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982803200:0@lo:38:kworker.0 00000100:00100000:1.0:1767319389.125862:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.125891:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982803456:0@lo:38:kworker.0 00000100:00100000:1.0:1767319389.125910:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.125927:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982803584:0@lo:38:kworker.0 00000100:00100000:1.0:1767319389.125951:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319389.125967:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982803968:0@lo:38:kworker.0 00000100:00100000:1.0:1767319389.125985:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319389.126174:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803200 00000100:00100000:0.0:1767319389.126182:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803200 00000100:00100000:2.0:1767319389.126252:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982803072:0@lo:400:kworker.0 00000100:00100000:3.0:1767319389.126350:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982803328:0@lo:400:kworker.0 00000100:00100000:3.0:1767319389.126380:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319389.128841:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803840 00000100:00100000:2.0:1767319389.128855:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803840 00000100:00100000:2.0:1767319389.128917:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803712 00000100:00100000:2.0:1767319389.128921:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803712 00000100:00100000:2.0:1767319389.128946:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 687 00000100:00100000:2.0:1767319389.128955:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982803840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.129042:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982803840:12345-0@lo:400:kworker.0 Request processed in 87us (3394us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319389.129059:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 687 00000100:00100000:2.0:1767319389.129074:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 688 00000100:00100000:2.0:1767319389.129080:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982803712:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319389.129108:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982803712:12345-0@lo:400:kworker.0 Request processed in 28us (3377us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319389.129113:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 688 00000100:00100000:2.0:1767319389.129131:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803584 00000100:00100000:2.0:1767319389.129136:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803584 00000100:00100000:2.0:1767319389.129154:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 382 00000100:00100000:2.0:1767319389.129159:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d87450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982803584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319389.129525:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.132546:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803456 00000100:00100000:0.0:1767319389.132562:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803456 00000100:00100000:0.0:1767319389.132603:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803968 00000100:00100000:0.0:1767319389.132606:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803968 00000100:00100000:0.0:1767319389.132617:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803328 00000100:00100000:0.0:1767319389.132620:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803328 00000100:00100000:0.0:1767319389.132664:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085 00000100:00100000:0.0:1767319389.132676:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982803328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319389.132775:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982803328:12345-0@lo:400:kworker.0 Request processed in 103us (6393us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319389.132782:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085 00000100:00100000:0.0:1767319389.132799:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1427 00000100:00100000:0.0:1767319389.132803:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.132821:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.132858:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803200:12345-0@lo:38:kworker.0 Request processed in 53us (6994us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.132864:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1427 00000100:00100000:0.0:1767319389.132875:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1428 00000100:00100000:0.0:1767319389.132878:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.132888:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.132911:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803456:12345-0@lo:38:kworker.0 Request processed in 31us (7000us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.132916:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1428 00000100:00100000:0.0:1767319389.132926:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1429 00000100:00100000:0.0:1767319389.132930:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319389.132938:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319389.132959:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982803968:12345-0@lo:38:kworker.0 Request processed in 28us (6973us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319389.132965:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1429 00000100:00100000:3.0:1767319389.133885:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767319389.133957:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f51500 x1853163982803328/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319405 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319389.133972:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982803328:0@lo:400:kworker.0 00000100:00100000:3.0:1767319389.134014:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982803072 00000100:00100000:3.0:1767319389.134018:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982803072 00000100:00100000:3.0:1767319389.134059:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086 00000100:00100000:3.0:1767319389.134066:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982803072:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319389.134119:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982803072:12345-0@lo:400:kworker.0 Request processed in 53us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319389.134126:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086 00010000:00080000:3.0:1767319389.134151:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f50000 x1853163982803072/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319405 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319389.134161:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982803072:0@lo:400:kworker.0 00000100:00080000:1.0:1767319389.138355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319389.138368:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319389.138372:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319389.138444:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982803712:0@lo:400:kworker.0 00000100:00100000:1.0:1767319389.138460:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982803840:0@lo:400:kworker.0 00000100:00080000:0.0:1767319389.138573:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.138579:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.138582:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982803200:0@lo:38:kworker.0 00000100:00080000:0.0:1767319389.138593:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.138596:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.138599:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982803456:0@lo:38:kworker.0 00000100:00080000:0.0:1767319389.138609:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.138612:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.138615:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982803968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319389.148610:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d87450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982803584:12345-0@lo:38:kworker.0 Request processed in 19444us (22652us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319389.148625:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 382 00000100:00080000:0.0:1767319389.148905:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319389.148913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319389.148917:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982803584:0@lo:38:kworker.0 00010000:00010000:1.0:1767319389.317090:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319390.341177:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319391.119388:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319391.119668:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319391.120937:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622400 00000100:00100000:0.0:1767319391.120950:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622400 00000100:00100000:2.0:1767319391.121025:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622272 00000100:00100000:2.0:1767319391.121030:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622272 00000100:00100000:2.0:1767319391.121070:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1087 00000100:00100000:2.0:1767319391.121081:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915622400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319391.121151:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915622400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 73us (1774us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319391.121161:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1430 00000100:00100000:0.0:1767319391.121173:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915622272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319391.121193:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319391.121222:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915622272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (1554us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319391.121229:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1430 00000100:00080000:1.0:1767319391.121328:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319391.121490:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1087 00010000:00010000:1.0:1767319391.365292:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319392.388435:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319392.708450:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319392.708470:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982804096:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319392.708526:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319392.708595:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804096 00000100:00100000:2.0:1767319392.708604:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804096 00000100:00100000:2.0:1767319392.708671:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 740 00000100:00100000:2.0:1767319392.708678:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982804096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319392.708703:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319392.708763:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982804096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (235us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319392.708770:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 740 00000100:00100000:3.0:1767319392.708896:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982804224:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319392.708930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319392.708970:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982804096:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319392.709116:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319392.709796:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804224 00000100:00100000:2.0:1767319392.709802:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804224 00000100:00100000:2.0:1767319392.709842:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 741 00000100:00100000:2.0:1767319392.709847:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982804224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319392.709864:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319392.709909:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982804224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (978us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319392.709914:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 741 00000100:00100000:1.0:1767319392.710204:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982804224:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319392.710259:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319393.417115:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319394.245509:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319394.245518:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319394.245579:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319394.245583:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319394.245587:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319394.245594:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4637 00000100:00080000:2.0:1767319394.245599:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319394.245605:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319394.245624:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4a80 x1853163982804480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319394.245625:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767319394.245649:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319394.245652:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319394.245654:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319394.245658:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4637 00000100:00100000:0.0:1767319394.245659:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767319394.245660:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319394.245663:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319394.245671:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982804608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.245673:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4700 x1853163982804736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319394.245680:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319394.245683:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319394.245684:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319394.245687:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4637 00000100:00080000:2.0:1767319394.245689:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319394.245692:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319394.245703:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5c00 x1853163982804864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319394.245707:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982804352:0@lo:400:kworker.0 00000100:00080000:2.0:1767319394.245738:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319394.245742:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319394.245744:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319394.245748:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4637 00000100:00080000:2.0:1767319394.245751:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319394.245759:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.245809:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982804992:0@lo:400:kworker.0 00000100:00100000:0.0:1767319394.245810:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319394.245824:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319394.245840:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7800 x1853163982805248/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319394.245848:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804352 00000100:00100000:3.0:1767319394.245850:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319394.245855:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804352 00000100:00100000:2.0:1767319394.245870:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804608 00000100:00100000:2.0:1767319394.245874:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804608 00000100:00100000:3.0:1767319394.245878:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319394.245888:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982805120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319394.245912:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.245944:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804992 00000100:00100000:3.0:1767319394.245948:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804992 00000100:00100000:1.0:1767319394.245949:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982804480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319394.245982:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.245984:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805120 00000100:00100000:3.0:1767319394.245989:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805120 00000100:00100000:1.0:1767319394.246005:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982804736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319394.246019:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 689 00000100:00100000:1.0:1767319394.246025:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.246032:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982804992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.246043:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982804864:0@lo:38:kworker.0 00000100:00100000:1.0:1767319394.246116:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.246126:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982804992:12345-0@lo:400:kworker.0 Request processed in 96us (278us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319394.246136:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982805248:0@lo:38:kworker.0 00000100:00100000:3.0:1767319394.246140:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 689 00000100:00100000:1.0:1767319394.246160:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319394.246164:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 690 00000100:00100000:3.0:1767319394.246174:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982805120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.246192:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804864 00000100:00100000:1.0:1767319394.246196:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804864 00000100:00100000:3.0:1767319394.246210:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982805120:12345-0@lo:400:kworker.0 Request processed in 38us (296us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319394.246218:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 383 00000100:00100000:3.0:1767319394.246219:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 690 00000100:00100000:1.0:1767319394.246227:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982804864:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319394.246371:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982805120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319394.246383:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982804992:0@lo:400:kworker.0 00010000:00020000:1.0:1767319394.246387:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319394.246420:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804480 00000100:00100000:1.0:1767319394.246426:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982804864:12345-0@lo:38:kworker.0 Request processed in 198us (309us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319394.246426:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804480 00000100:00100000:1.0:1767319394.246435:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 383 00000100:00100000:0.0:1767319394.246474:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982804736 00000100:00100000:0.0:1767319394.246477:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982804736 00000100:00100000:0.0:1767319394.246486:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805248 00000100:00100000:0.0:1767319394.246488:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805248 00000100:00100000:0.0:1767319394.246496:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088 00000100:00100000:0.0:1767319394.246502:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982804608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.246896:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089 00000100:00100000:1.0:1767319394.246904:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982804352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319394.247474:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982804352:12345-0@lo:400:kworker.0 Request processed in 571us (1715us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319394.247481:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089 00000100:00100000:1.0:1767319394.247496:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1431 00000100:00100000:1.0:1767319394.247500:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982804736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319394.247512:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319394.247544:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982804736:12345-0@lo:38:kworker.0 Request processed in 33us (1509us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319394.247550:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1431 00000100:00100000:1.0:1767319394.247560:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1432 00000100:00100000:1.0:1767319394.247564:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982805248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319394.247572:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319394.247590:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982805248:12345-0@lo:38:kworker.0 Request processed in 25us (1432us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319394.247595:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1432 00010000:00080000:1.0:1767319394.247619:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5180 x1853163982804352/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319410 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319394.247629:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982804352:0@lo:400:kworker.0 00000100:00080000:1.0:1767319394.247652:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.247658:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.247661:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982804736:0@lo:38:kworker.0 00000100:00080000:1.0:1767319394.247669:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.247672:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.247675:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982804864:0@lo:38:kworker.0 00000100:00080000:1.0:1767319394.247683:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.247685:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.247688:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982805248:0@lo:38:kworker.0 00000100:00080000:1.0:1767319394.248455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319394.248461:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319394.248463:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319394.248622:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982804608:12345-0@lo:400:kworker.0 Request processed in 2118us (2812us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319394.248631:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088 00010000:00080000:1.0:1767319394.248821:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5880 x1853163982804608/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319410 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319394.248832:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982804608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319394.249830:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1433 00000100:00100000:2.0:1767319394.249841:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982804480:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319394.249860:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319394.249969:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982804480:12345-0@lo:38:kworker.0 Request processed in 126us (3986us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319394.249977:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1433 00000100:00080000:1.0:1767319394.254897:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319394.254909:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319394.254914:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982804480:0@lo:38:kworker.0 00010000:00010000:1.0:1767319394.437929:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319395.461252:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319396.237552:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319396.237617:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319396.239838:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622656 00000100:00100000:2.0:1767319396.239854:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622656 00000100:00100000:0.0:1767319396.239986:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622528 00000100:00100000:0.0:1767319396.239992:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622528 00000100:00100000:0.0:1767319396.240024:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1090 00000100:00100000:0.0:1767319396.240035:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915622656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319396.240114:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915622656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (2571us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319396.240123:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1090 00000100:00100000:0.0:1767319396.240141:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1434 00000100:00100000:0.0:1767319396.240146:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915622528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319396.240164:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319396.240205:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915622528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (2586us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319396.240211:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1434 00000100:00080000:1.0:1767319396.240521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319396.484488:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319397.508518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319397.828934:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319397.828959:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982805376:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319397.829040:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319397.829086:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982805504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319397.829109:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319397.829326:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805376 00000100:00100000:0.0:1767319397.829334:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805376 00000100:00100000:0.0:1767319397.829401:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805504 00000100:00100000:0.0:1767319397.829405:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805504 00000100:00100000:0.0:1767319397.829423:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 742 00000100:00100000:0.0:1767319397.829431:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982805376:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319397.829461:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319397.829491:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00100000:3.0:1767319397.829499:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982805504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319397.829515:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767319397.829922:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319397.829927:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319397.838287:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982805504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 8762us (9150us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319397.838303:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00100000:2.0:1767319397.839725:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982805504:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319397.841312:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982805376:12345-0@lo:13:osp-pre-0-1.0 Request processed in 11873us (12263us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319397.841326:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 742 00000100:00100000:3.0:1767319397.841425:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982805376:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319398.532512:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319399.364503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319399.364515:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319399.364585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319399.364589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319399.364646:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319399.364655:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4642 00000100:00080000:3.0:1767319399.364660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319399.364667:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319399.364689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51500 x1853163982805760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319399.364724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319399.364727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319399.364729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319399.364733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4642 00000100:00080000:3.0:1767319399.364735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319399.364739:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319399.364749:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cd880 x1853163982806016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319399.364757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319399.364760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319399.364761:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319399.364765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4642 00000100:00080000:3.0:1767319399.364767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319399.364770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319399.364778:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf480 x1853163982806144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319399.364818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319399.364822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319399.364823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319399.364826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4642 00000100:00080000:3.0:1767319399.364829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319399.364832:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319399.364841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cfb80 x1853163982806528/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319399.364865:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982806272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319399.364903:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982805888:0@lo:400:kworker.0 00000100:00100000:0.0:1767319399.364955:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982805632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319399.364966:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982805760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319399.365208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319399.365305:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ce680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982806400:0@lo:400:kworker.0 00000100:00100000:1.0:1767319399.365330:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319399.365364:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806400 00000100:00100000:1.0:1767319399.365372:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806400 00000100:00100000:1.0:1767319399.365424:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 691 00000100:00100000:3.0:1767319399.365686:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319399.365720:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806272 00000100:00100000:3.0:1767319399.365724:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806272 00000100:00100000:3.0:1767319399.365751:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 692 00000100:00100000:3.0:1767319399.365758:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982806272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.365809:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982806272:12345-0@lo:400:kworker.0 Request processed in 52us (123us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319399.365816:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 692 00000100:00100000:3.0:1767319399.365853:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982806272:0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.365876:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805888 00000100:00100000:3.0:1767319399.365880:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805888 00000100:00100000:3.0:1767319399.365909:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091 00000100:00100000:3.0:1767319399.365915:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982805888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.365947:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982805888:12345-0@lo:400:kworker.0 Request processed in 33us (740us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319399.365956:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091 00000100:00080000:3.0:1767319399.365985:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319399.365986:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982806400:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319399.365989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319399.365993:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:3.0:1767319399.366034:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059ce300 x1853163982805888/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319415 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319399.366040:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982806400:12345-0@lo:400:kworker.0 Request processed in 605us (706us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319399.366054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982805888:0@lo:400:kworker.0 00000100:00100000:1.0:1767319399.366057:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 691 00000100:00100000:3.0:1767319399.366093:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ce680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982806400:0@lo:400:kworker.0 00000100:00100000:0.0:1767319399.367002:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319399.367699:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805632 00000100:00100000:3.0:1767319399.367708:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805632 00000100:00100000:3.0:1767319399.367769:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092 00000100:00100000:3.0:1767319399.367776:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982805632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319399.367826:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982805632:12345-0@lo:400:kworker.0 Request processed in 50us (828us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319399.367831:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092 00010000:00080000:0.0:1767319399.368575:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f50000 x1853163982805632/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319415 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319399.368596:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982805632:0@lo:400:kworker.0 00000100:00080000:1.0:1767319399.368683:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319399.368751:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319399.368785:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982806016:0@lo:38:kworker.0 00000100:00100000:0.0:1767319399.368795:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982805760 00000100:00100000:0.0:1767319399.368802:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982805760 00000100:00100000:2.0:1767319399.368817:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319399.370290:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982806144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319399.370295:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806016 00000100:00100000:0.0:1767319399.370304:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806016 00000100:00100000:2.0:1767319399.370337:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319399.370373:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982806528:0@lo:38:kworker.0 00000100:00100000:3.0:1767319399.370378:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1435 00000100:00100000:3.0:1767319399.370389:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982805760:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319399.370396:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767319399.370411:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319399.370436:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806144 00000100:00100000:2.0:1767319399.370450:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806144 00000100:00100000:2.0:1767319399.370475:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 384 00000100:00100000:2.0:1767319399.370480:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982806144:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319399.370488:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982805760:12345-0@lo:38:kworker.0 Request processed in 96us (1734us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767319399.370495:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319399.370499:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1435 00000100:00100000:3.0:1767319399.370519:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806528 00000100:00100000:3.0:1767319399.370524:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806528 00000100:00100000:2.0:1767319399.370525:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982806144:12345-0@lo:38:kworker.0 Request processed in 43us (187us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319399.370533:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 384 00000100:00100000:3.0:1767319399.370552:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1436 00000100:00100000:3.0:1767319399.370559:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982806016:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767319399.370569:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319399.370575:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319399.370578:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982805760:0@lo:38:kworker.0 00000100:00080000:2.0:1767319399.370586:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319399.370590:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:3.0:1767319399.370604:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319399.370631:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982806016:12345-0@lo:38:kworker.0 Request processed in 72us (1814us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319399.370637:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1436 00000100:00100000:3.0:1767319399.370648:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1437 00000100:00100000:3.0:1767319399.370652:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982806528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319399.370661:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319399.370681:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982806528:12345-0@lo:38:kworker.0 Request processed in 28us (285us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319399.370687:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1437 00000100:00100000:2.0:1767319399.373740:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982806144:0@lo:38:kworker.0 00000100:00080000:2.0:1767319399.373778:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319399.373782:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319399.373785:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982806528:0@lo:38:kworker.0 00000100:00080000:2.0:1767319399.373812:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319399.373815:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319399.373818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982806016:0@lo:38:kworker.0 00010000:00010000:1.0:1767319399.556525:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319400.580562:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319401.355791:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319401.355916:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622912 00000100:00100000:0.0:1767319401.355925:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622912 00000100:00100000:0.0:1767319401.356080:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1093 00000100:00100000:0.0:1767319401.356091:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915622912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319401.356166:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319401.356182:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915622912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (400us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319401.356191:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1093 00000100:00100000:0.0:1767319401.356220:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915622784 00000100:00100000:0.0:1767319401.356223:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915622784 00000100:00100000:0.0:1767319401.356243:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1438 00000100:00100000:0.0:1767319401.356248:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915622784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319401.356309:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319401.356337:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915622784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319401.356344:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1438 00000100:00080000:1.0:1767319401.357424:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319401.606812:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319402.631509:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319402.948544:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982806656:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319402.948621:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319402.948835:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982806784:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319402.948868:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319402.948947:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806656 00000100:00100000:1.0:1767319402.948956:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806656 00000100:00100000:1.0:1767319402.949036:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806784 00000100:00100000:1.0:1767319402.949040:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806784 00000100:00100000:1.0:1767319402.949059:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00100000:1.0:1767319402.949068:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982806656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319402.949097:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319402.949139:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982806656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (517us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319402.949146:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00000100:00100000:1.0:1767319402.949163:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 745 00000100:00100000:1.0:1767319402.949169:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982806784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319402.949184:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319402.949216:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982806784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319402.949222:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 745 00000100:00100000:1.0:1767319402.949263:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982806784:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319402.949537:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319402.949541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319402.949644:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982806656:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319403.652506:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319404.484708:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319404.484719:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319404.484785:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319404.484790:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319404.484793:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319404.484802:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4647 00000100:00080000:2.0:1767319404.484807:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319404.484814:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319404.484814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982806912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319404.484836:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4700 x1853163982807040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319404.484870:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319404.484873:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319404.484875:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319404.484878:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4647 00000100:00080000:2.0:1767319404.484881:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319404.484883:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319404.484887:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.484893:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6a00 x1853163982807296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319404.484901:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319404.484904:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319404.484906:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319404.484909:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4647 00000100:00080000:2.0:1767319404.484911:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319404.484916:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319404.484922:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982807168:0@lo:400:kworker.0 00000100:00100000:2.0:1767319404.484938:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163982807424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319404.484965:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319404.484976:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319404.484980:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319404.484982:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319404.484987:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4647 00000100:00080000:2.0:1767319404.484989:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319404.484992:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319404.485004:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163982807808/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319404.485014:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982807552:0@lo:400:kworker.0 00000100:00100000:2.0:1767319404.485048:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982807680:0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.485108:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319404.485139:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807552 00000100:00100000:0.0:1767319404.485146:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807552 00000100:00100000:0.0:1767319404.485187:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 693 00000100:00100000:0.0:1767319404.485193:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982807552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.485231:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982807552:12345-0@lo:400:kworker.0 Request processed in 38us (122us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319404.485237:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 693 00000100:00080000:0.0:1767319404.485255:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319404.485295:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982807552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.485321:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982807040:0@lo:38:kworker.0 00000100:00100000:0.0:1767319404.485355:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319404.486050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319404.486582:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982806912 00000100:00100000:3.0:1767319404.486590:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982806912 00000100:00100000:0.0:1767319404.486624:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982807296:0@lo:38:kworker.0 00000100:00100000:3.0:1767319404.486644:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807168 00000100:00100000:3.0:1767319404.486648:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807168 00000100:00100000:0.0:1767319404.486663:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319404.486674:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807040 00000100:00100000:3.0:1767319404.486679:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807040 00000100:00100000:0.0:1767319404.486687:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982807424:0@lo:38:kworker.0 00000100:00100000:3.0:1767319404.486692:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807296 00000100:00100000:3.0:1767319404.486697:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807296 00000100:00100000:3.0:1767319404.486711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094 00000100:00100000:0.0:1767319404.486718:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319404.486727:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982806912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.486747:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982807808:0@lo:38:kworker.0 00000100:00100000:0.0:1767319404.486785:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319404.486786:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982806912:12345-0@lo:400:kworker.0 Request processed in 59us (1894us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319404.486796:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094 00000100:00100000:3.0:1767319404.486816:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807808 00000100:00100000:0.0:1767319404.486818:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807424 00000100:00100000:3.0:1767319404.486820:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807808 00000100:00100000:0.0:1767319404.486824:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807424 00000100:00100000:3.0:1767319404.486837:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767319404.486846:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982807168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319404.486854:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 385 00000100:00100000:0.0:1767319404.486864:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982807424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319404.486884:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319404.486890:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982807168:12345-0@lo:400:kworker.0 Request processed in 43us (1924us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319404.486898:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767319404.486915:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1439 00000100:00100000:3.0:1767319404.486920:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982807040:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319404.486925:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982807424:12345-0@lo:38:kworker.0 Request processed in 60us (209us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767319404.486936:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319404.486937:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 385 00000100:00100000:0.0:1767319404.486965:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807680 00000100:00100000:0.0:1767319404.486969:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807680 00000100:00100000:3.0:1767319404.486972:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982807040:12345-0@lo:38:kworker.0 Request processed in 49us (1624us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319404.486981:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1439 00000100:00100000:3.0:1767319404.486999:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1440 00000100:00100000:0.0:1767319404.487003:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 694 00000100:00100000:3.0:1767319404.487008:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982807296:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319404.487015:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982807680:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767319404.487020:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319404.487054:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982807680:12345-0@lo:400:kworker.0 Request processed in 42us (1005us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319404.487061:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 694 00000100:00100000:0.0:1767319404.487092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982807680:0@lo:400:kworker.0 00000100:00080000:0.0:1767319404.487114:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319404.487119:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319404.487121:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982807040:0@lo:38:kworker.0 00000100:00080000:0.0:1767319404.487130:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319404.487133:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319404.487136:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982807424:0@lo:38:kworker.0 00000100:00080000:0.0:1767319404.487166:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319404.487170:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319404.487182:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1441 00000100:00100000:0.0:1767319404.487186:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982807808:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319404.487195:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319404.487222:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982807808:12345-0@lo:38:kworker.0 Request processed in 35us (442us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319404.487228:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1441 00000100:00080000:0.0:1767319404.487251:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319404.487255:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319404.487257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982807808:0@lo:38:kworker.0 00010000:00080000:0.0:1767319404.487317:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5c00 x1853163982806912/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319420 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319404.487328:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982806912:0@lo:400:kworker.0 00010000:00080000:2.0:1767319404.487444:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5180 x1853163982807168/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319420 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319404.487454:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982807168:0@lo:400:kworker.0 00000100:00100000:3.0:1767319404.489534:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982807296:12345-0@lo:38:kworker.0 Request processed in 2517us (2863us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319404.489543:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1440 00000100:00080000:0.0:1767319404.489700:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319404.489706:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319404.489709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982807296:0@lo:38:kworker.0 00010000:00010000:1.0:1767319404.676614:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319405.701921:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319406.479330:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319406.479381:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319406.479595:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623168 00000100:00100000:3.0:1767319406.479607:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623168 00000100:00080000:0.0:1767319406.479760:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319406.479880:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623040 00000100:00100000:3.0:1767319406.479885:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623040 00000100:00100000:2.0:1767319406.480074:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1096 00000100:00100000:2.0:1767319406.480086:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915623168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319406.480219:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915623168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 134us (907us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319406.480227:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1096 00000100:00100000:2.0:1767319406.480249:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1442 00000100:00100000:2.0:1767319406.480254:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319406.480321:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319406.480358:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (976us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319406.480364:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1442 00010000:00010000:1.0:1767319406.724613:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319407.748435:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319408.068801:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982807936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319408.068891:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319408.069052:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319408.070240:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982807936 00000100:00100000:3.0:1767319408.070253:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982807936 00000100:00100000:2.0:1767319408.070325:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319408.070421:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808064 00000100:00100000:1.0:1767319408.070426:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808064 00000100:00100000:1.0:1767319408.070486:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00100000:1.0:1767319408.070495:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319408.070534:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319408.070558:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000100:00100000:3.0:1767319408.070566:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982807936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319408.070581:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319408.070589:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319408.070596:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00000100:00100000:3.0:1767319408.070610:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982807936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (1720us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319408.070615:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00000100:00100000:2.0:1767319408.070803:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808064:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319408.071557:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319408.071563:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319408.071597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982807936:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319408.777789:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319409.604400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.604409:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319409.604666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.604672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.604676:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.604684:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.604688:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319409.604693:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982808192:0@lo:400:kworker.0 00000100:00080000:1.0:1767319409.604695:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319409.604719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582300 x1853163982808320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319409.604741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.604743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.604746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.604749:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.604751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.604755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319409.604765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd500 x1853163982808576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319409.604774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.604777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.604778:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.604781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.604783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.604786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319409.604796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd880 x1853163982808704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319409.604843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319409.604846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319409.604848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319409.604851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4653 00000100:00080000:1.0:1767319409.604854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319409.604857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319409.604866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163982809088/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319409.604915:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982808320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319409.604975:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319409.605016:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808320 00000100:00100000:1.0:1767319409.605024:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808320 00000100:00100000:1.0:1767319409.605060:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1443 00000100:00100000:1.0:1767319409.605067:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982808320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319409.605087:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319409.605119:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319409.605121:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982808320:12345-0@lo:38:kworker.0 Request processed in 55us (147us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319409.605127:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1443 00000100:00100000:1.0:1767319409.605141:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808192 00000100:00100000:1.0:1767319409.605145:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808192 00000100:00100000:2.0:1767319409.605162:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319409.605167:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982808448:0@lo:400:kworker.0 00000100:00100000:1.0:1767319409.605178:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097 00000100:00100000:1.0:1767319409.605183:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982808192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.605188:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.605214:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808960:0@lo:400:kworker.0 00000100:00100000:1.0:1767319409.605219:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982808192:12345-0@lo:400:kworker.0 Request processed in 36us (101us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319409.605225:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097 00000100:00100000:1.0:1767319409.605235:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808448 00000100:00100000:1.0:1767319409.605238:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808448 00000100:00100000:2.0:1767319409.605242:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319409.605266:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098 00000100:00100000:1.0:1767319409.605298:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982808448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.605301:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319409.605306:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808832:0@lo:400:kworker.0 00000100:00100000:1.0:1767319409.605324:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982808448:12345-0@lo:400:kworker.0 Request processed in 52us (135us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319409.605324:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319409.605329:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098 00000100:00100000:2.0:1767319409.605350:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808960 00000100:00100000:2.0:1767319409.605353:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808960 00000100:00080000:1.0:1767319409.605360:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319409.605364:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319409.605368:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982808320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319409.605374:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982808576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.605376:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808832 00000100:00100000:2.0:1767319409.605378:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808832 00000100:00100000:1.0:1767319409.605396:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.605396:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 695 00000100:00100000:2.0:1767319409.605402:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639349050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319409.605415:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982808704:0@lo:38:kworker.0 00000100:00100000:2.0:1767319409.605433:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639349050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808960:12345-0@lo:400:kworker.0 Request processed in 31us (189us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319409.605437:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319409.605439:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 695 00000100:00100000:1.0:1767319409.605456:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982809088:0@lo:38:kworker.0 00000100:00100000:1.0:1767319409.605472:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319409.605498:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808576 00000100:00100000:1.0:1767319409.605502:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808576 00000100:00100000:1.0:1767319409.605510:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809088 00000100:00100000:1.0:1767319409.605512:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809088 00000100:00100000:1.0:1767319409.605520:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1444 00000100:00100000:1.0:1767319409.605523:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982808576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319409.605532:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319409.605555:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982808576:12345-0@lo:38:kworker.0 Request processed in 30us (158us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319409.605560:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1444 00000100:00100000:1.0:1767319409.605568:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1445 00000100:00100000:1.0:1767319409.605571:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982809088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319409.605579:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319409.605595:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982809088:12345-0@lo:38:kworker.0 Request processed in 23us (122us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319409.605600:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1445 00000100:00080000:1.0:1767319409.605619:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319409.605622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319409.605625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982808576:0@lo:38:kworker.0 00000100:00080000:1.0:1767319409.605632:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319409.605634:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319409.605637:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982809088:0@lo:38:kworker.0 00000100:00080000:1.0:1767319409.605660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319409.605672:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00100000:2.0:1767319409.605679:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808832:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.605722:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982808832:12345-0@lo:400:kworker.0 Request processed in 44us (397us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319409.605728:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00100000:2.0:1767319409.605765:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808832:0@lo:400:kworker.0 00000100:00100000:2.0:1767319409.605784:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982808960:0@lo:400:kworker.0 00010000:00080000:3.0:1767319409.605795:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382de680 x1853163982808448/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319425 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319409.605805:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982808448:0@lo:400:kworker.0 00010000:00080000:3.0:1767319409.605814:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a580e00 x1853163982808192/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319425 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319409.605822:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982808192:0@lo:400:kworker.0 00000100:00100000:0.0:1767319409.606130:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982808704 00000100:00100000:0.0:1767319409.606135:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982808704 00000100:00100000:0.0:1767319409.606165:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 386 00000100:00100000:0.0:1767319409.606170:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982808704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319409.606180:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319409.606217:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982808704:12345-0@lo:38:kworker.0 Request processed in 46us (778us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319409.606223:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 386 00000100:00080000:1.0:1767319409.606297:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319409.606301:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319409.606305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982808704:0@lo:38:kworker.0 00010000:00010000:1.0:1767319409.796677:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319410.821673:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319411.596944:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319411.597028:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623424 00000100:00100000:3.0:1767319411.597038:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623424 00000100:00100000:3.0:1767319411.597121:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1099 00000100:00100000:3.0:1767319411.597132:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915623424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319411.597223:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915623424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319411.597231:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1099 00000100:00080000:1.0:1767319411.597601:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319411.600315:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319411.600347:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915623552 offset 0 length 520: 4 00000100:00100000:2.0:1767319411.600425:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623296 00000100:00100000:2.0:1767319411.600443:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623296 00000100:00100000:2.0:1767319411.600484:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1446 00000100:00100000:2.0:1767319411.600491:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319411.600513:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319411.600576:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 84us (269us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319411.600583:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1446 00010000:00010000:1.0:1767319411.845074:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319412.869106:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319413.188907:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982809216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319413.188945:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982809344:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319413.188981:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319413.189459:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809216 00000100:00100000:2.0:1767319413.189467:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809216 00000100:00100000:2.0:1767319413.189574:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 748 00000100:00080000:1.0:1767319413.189584:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319413.189585:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982809216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319413.189626:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319413.189689:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982809216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 106us (707us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319413.189696:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 748 00000100:00100000:0.0:1767319413.189759:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319413.189811:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982809216:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319413.191388:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809344 00000100:00100000:3.0:1767319413.191401:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809344 00000100:00080000:0.0:1767319413.191460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319413.193130:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 749 00000100:00100000:3.0:1767319413.193145:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982809344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319413.193177:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319413.193262:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982809344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (3502us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319413.193325:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982809344:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319413.193368:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 749 00010000:00010000:1.0:1767319413.892626:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319414.724419:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319389/real 1767319389] req@ffff934610f52680 x1853163982802944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319414 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319414.724456:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934610f52680 x1853163982802944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319414 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319414.724478:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319414.724486:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319414.724490:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982802944:0@lo:250:kworker.0 00000100:00080000:3.0:1767319414.724641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319414.724647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319414.724650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319414.724655:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4637 00000100:00080000:3.0:1767319414.724659:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319414.724665:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319414.724697:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc380 x1853163982809472/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319414.724756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319414.724760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319414.724761:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319414.724765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319414.724767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319414.724770:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319414.724783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cca80 x1853163982809728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319414.724798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319414.724801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319414.724804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319414.724807:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319414.724809:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319414.724813:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319414.724829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cdf80 x1853163982809984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319414.724838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319414.724841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319414.724843:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767319414.724848:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982809600:0@lo:400:kworker.0 00000100:00080000:3.0:1767319414.724850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4658 00000100:00080000:3.0:1767319414.724855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319414.724860:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319414.724873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf100 x1853163982810112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319414.724910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319414.724914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319414.724916:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319414.724921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4658 00000100:00100000:0.0:1767319414.724923:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319414.724925:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319414.724931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319414.724943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cea00 x1853163982810496/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319414.724973:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809600 00000100:00100000:3.0:1767319414.724980:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809600 00000100:00100000:0.0:1767319414.724993:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319414.725001:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982809856:0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725028:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319414.725037:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809856 00000100:00100000:3.0:1767319414.725042:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809856 00000100:00080000:0.0:1767319414.725056:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319414.725071:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982810368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319414.725072:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100 00000100:00100000:3.0:1767319414.725086:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982809600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725105:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319414.725131:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319414.725136:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982810240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319414.725149:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982809600:12345-0@lo:400:kworker.0 Request processed in 65us (227us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319414.725168:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319414.725192:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810368 00000100:00100000:0.0:1767319414.725195:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810368 00000100:00100000:0.0:1767319414.725212:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810240 00000100:00100000:0.0:1767319414.725214:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810240 00000100:00100000:0.0:1767319414.725224:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 697 00000100:00100000:0.0:1767319414.725228:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982810368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725262:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982810368:12345-0@lo:400:kworker.0 Request processed in 24us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319414.725267:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 697 00000100:00100000:0.0:1767319414.725295:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767319414.725299:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982810240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725319:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982810240:12345-0@lo:400:kworker.0 Request processed in 19us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319414.725322:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000100:00100000:3.0:1767319414.725332:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100 00010000:00080000:0.0:1767319414.725347:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cc700 x1853163982809600/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319414.725356:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101 00000100:00100000:0.0:1767319414.725359:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982809600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319414.725367:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982809856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725384:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982810240:0@lo:400:kworker.0 00000100:00100000:0.0:1767319414.725392:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982810368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319414.725418:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982809856:12345-0@lo:400:kworker.0 Request processed in 53us (387us total) trans 0 rc 0/0 00010000:00080000:0.0:1767319414.725425:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cce00 x1853163982809856/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319414.725428:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101 00000100:00100000:0.0:1767319414.725438:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982809856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319414.725607:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982809472:0@lo:250:kworker.0 00000400:00000100:1.0:1767319414.725633:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982809472 offset 0 length 520: 4 00000100:00100000:1.0:1767319414.725654:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982809728:0@lo:38:kworker.0 00000100:00100000:1.0:1767319414.725677:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319414.725705:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982809984:0@lo:38:kworker.0 00000100:00100000:1.0:1767319414.725727:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319414.725752:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982810112:0@lo:38:kworker.0 00000100:00100000:1.0:1767319414.725774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319414.725793:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982810496:0@lo:38:kworker.0 00000100:00100000:1.0:1767319414.725813:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319414.725842:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810112 00000100:00100000:1.0:1767319414.725846:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810112 00000100:00100000:1.0:1767319414.725868:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 387 00000100:00100000:1.0:1767319414.725873:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982810112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319414.725891:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319414.725923:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982810112:12345-0@lo:38:kworker.0 Request processed in 48us (147us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319414.725929:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 387 00000100:00080000:1.0:1767319414.725957:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319414.725961:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319414.725988:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982810112:0@lo:38:kworker.0 00000100:00100000:0.0:1767319414.726055:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809728 00000100:00100000:0.0:1767319414.726060:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809728 00000100:00100000:0.0:1767319414.726085:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982809984 00000100:00100000:0.0:1767319414.726088:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982809984 00000100:00100000:0.0:1767319414.726099:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810496 00000100:00100000:0.0:1767319414.726102:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810496 00000100:00100000:0.0:1767319414.726111:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1447 00000100:00100000:0.0:1767319414.726115:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982809728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319414.726125:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319414.726168:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982809728:12345-0@lo:38:kworker.0 Request processed in 40us (479us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319414.726175:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1447 00000100:00100000:0.0:1767319414.726187:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1448 00000100:00100000:0.0:1767319414.726191:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982809984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319414.726200:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319414.726224:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982809984:12345-0@lo:38:kworker.0 Request processed in 31us (495us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319414.726230:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1448 00000100:00100000:0.0:1767319414.726240:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1449 00000100:00100000:0.0:1767319414.726243:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982810496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319414.726251:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319414.726292:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982810496:12345-0@lo:38:kworker.0 Request processed in 47us (478us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319414.726298:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1449 00000100:00080000:1.0:1767319414.727147:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319414.727154:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319414.727169:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982809728:0@lo:38:kworker.0 00000100:00080000:1.0:1767319414.727180:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319414.727183:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319414.727185:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982809984:0@lo:38:kworker.0 00000100:00080000:1.0:1767319414.727193:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319414.727196:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319414.727198:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982810496:0@lo:38:kworker.0 00010000:00010000:1.0:1767319414.917204:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319415.940511:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319416.717586:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319416.717676:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623808 00000100:00100000:1.0:1767319416.717686:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623808 00000100:00100000:1.0:1767319416.717764:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1102 00000100:00100000:1.0:1767319416.717776:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915623808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319416.717851:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915623808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319416.717859:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1102 00000100:00080000:0.0:1767319416.717969:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319416.717991:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319416.718801:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623680 00000100:00100000:1.0:1767319416.718808:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623680 00000100:00100000:1.0:1767319416.718848:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1450 00000100:00100000:1.0:1767319416.718853:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319416.718873:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319416.718928:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915623680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 73us (939us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319416.718936:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1450 00010000:00010000:1.0:1767319416.965078:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319417.988539:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319418.309574:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982810624:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319418.309646:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319418.309839:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810624 00000100:00100000:1.0:1767319418.309848:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810624 00000100:00100000:1.0:1767319418.309913:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00100000:1.0:1767319418.309923:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982810624:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319418.309950:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319418.310005:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982810624:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319418.310013:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00080000:0.0:1767319418.310133:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319418.310172:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982810624:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319418.311593:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982810752:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319418.311642:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319418.311733:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810752 00000100:00100000:3.0:1767319418.311740:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810752 00000100:00100000:3.0:1767319418.311792:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 751 00000100:00100000:3.0:1767319418.311802:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982810752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319418.311825:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319418.311861:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982810752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (219us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319418.311868:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 751 00000100:00080000:0.0:1767319418.311925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319418.311952:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982810752:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319419.012638:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319419.844541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319419.844553:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319419.844618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319419.844623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319419.844628:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319419.844637:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4663 00000100:00080000:3.0:1767319419.844643:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319419.844649:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319419.844671:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf480 x1853163982811008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319419.844703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319419.844707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319419.844709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319419.844713:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4663 00000100:00080000:3.0:1767319419.844716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319419.844720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319419.844730:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982811136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319419.844733:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cdf80 x1853163982811264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319419.844745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319419.844748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319419.844749:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319419.844753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4663 00000100:00080000:3.0:1767319419.844755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319419.844759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319419.844772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cca80 x1853163982811392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319419.844803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319419.844820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319419.844825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319419.844828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319419.844839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4663 00000100:00100000:2.0:1767319419.844861:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811136 00000100:00100000:2.0:1767319419.844869:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811136 00000100:00100000:1.0:1767319419.844906:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982810880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319419.844921:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103 00000100:00100000:2.0:1767319419.844931:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982811136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319419.844944:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319419.844986:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982811136:12345-0@lo:400:kworker.0 Request processed in 56us (183us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319419.844992:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982811520:0@lo:400:kworker.0 00000100:00100000:2.0:1767319419.845005:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103 00000100:00100000:2.0:1767319419.845019:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982810880 00000100:00100000:2.0:1767319419.845022:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982810880 00000100:00100000:2.0:1767319419.845041:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104 00000100:00100000:2.0:1767319419.845045:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982810880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319419.845086:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982810880:12345-0@lo:400:kworker.0 Request processed in 40us (143us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319419.845092:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104 00010000:00080000:2.0:1767319419.845131:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cfb80 x1853163982810880/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319435 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319419.845144:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982810880:0@lo:400:kworker.0 00010000:00080000:2.0:1767319419.845179:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cea00 x1853163982811136/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319435 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319419.845187:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982811136:0@lo:400:kworker.0 00000100:00100000:1.0:1767319419.845364:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319419.845404:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319419.845411:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982811648:0@lo:400:kworker.0 00000100:00100000:1.0:1767319419.845435:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319419.845463:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811520 00000100:00100000:1.0:1767319419.845467:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811520 00000100:00080000:0.0:1767319419.845492:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319419.845501:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811648 00000100:00100000:1.0:1767319419.845505:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811648 00000100:00100000:0.0:1767319419.845514:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000100:00100000:0.0:1767319419.845527:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982811520:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319419.845534:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 700 00000100:00100000:1.0:1767319419.845544:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982811648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319419.845576:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982811520:12345-0@lo:400:kworker.0 Request processed in 53us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319419.845584:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000100:00100000:1.0:1767319419.845591:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982811648:12345-0@lo:400:kworker.0 Request processed in 48us (154us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319419.845600:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 700 00000100:00100000:0.0:1767319419.845628:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982811008:0@lo:38:kworker.0 00000100:00100000:1.0:1767319419.845647:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982811648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319419.845661:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319419.845663:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982811520:0@lo:400:kworker.0 00000100:00100000:1.0:1767319419.845698:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811008 00000100:00100000:1.0:1767319419.845702:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811008 00000100:00100000:1.0:1767319419.845723:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1451 00000100:00100000:1.0:1767319419.845728:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982811008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319419.845745:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319419.845755:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982811264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319419.845780:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982811008:12345-0@lo:38:kworker.0 Request processed in 49us (119us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319419.845787:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1451 00000100:00100000:0.0:1767319419.845792:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319419.845880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982811392:0@lo:38:kworker.0 00000100:00100000:0.0:1767319419.845904:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319419.845931:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319419.845936:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319419.845939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982811008:0@lo:38:kworker.0 00000100:00100000:0.0:1767319419.845964:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811392 00000100:00080000:3.0:1767319419.845967:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319419.845969:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811392 00000100:00080000:3.0:1767319419.845975:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319419.845982:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811264 00000100:00100000:1.0:1767319419.845988:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811264 00000100:00100000:0.0:1767319419.845998:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 388 00000100:00100000:0.0:1767319419.846003:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982811392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319419.846014:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319419.846043:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982811392:12345-0@lo:38:kworker.0 Request processed in 39us (138us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319419.846050:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 388 00000100:00080000:0.0:1767319419.846083:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319419.846087:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319419.846090:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982811392:0@lo:38:kworker.0 00000100:00100000:1.0:1767319419.846214:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1452 00000100:00100000:1.0:1767319419.846220:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982811264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319419.846231:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319419.846267:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982811264:12345-0@lo:38:kworker.0 Request processed in 45us (477us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319419.846290:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1452 00000100:00100000:3.0:1767319419.846345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cd180 x1853163982811776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319419.846609:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982811776:0@lo:38:kworker.0 00000100:00100000:3.0:1767319419.846644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319419.848289:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319419.848300:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319419.848304:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982811264:0@lo:38:kworker.0 00000100:00100000:3.0:1767319419.848343:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811776 00000100:00100000:3.0:1767319419.848349:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811776 00000100:00100000:3.0:1767319419.848383:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1453 00000100:00100000:3.0:1767319419.848390:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982811776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319419.848409:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319419.848462:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982811776:12345-0@lo:38:kworker.0 Request processed in 71us (1816us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319419.848468:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1453 00000100:00080000:3.0:1767319419.848497:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319419.848501:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319419.848504:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982811776:0@lo:38:kworker.0 00010000:00010000:1.0:1767319420.039494:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319421.060521:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319421.838801:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319421.838976:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319421.840115:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915623936 00000100:00100000:1.0:1767319421.840115:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624064 00000100:00100000:0.0:1767319421.840128:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915623936 00000100:00100000:0.0:1767319421.840221:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1454 00000100:00100000:0.0:1767319421.840231:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915623936:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:1.0:1767319421.840233:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624064 00010000:00020000:0.0:1767319421.840255:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319421.840338:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915623936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 107us (1361us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319421.840354:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1454 00000100:00080000:3.0:1767319421.840358:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319421.840373:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1105 00000100:00100000:0.0:1767319421.840382:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915624064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319421.840419:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915624064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 40us (1631us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319421.840425:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1105 00010000:00010000:1.0:1767319422.084601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319423.108429:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319423.428783:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319423.428800:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982811904:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319423.428859:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319423.428909:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982811904 00000100:00100000:2.0:1767319423.428915:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982811904 00000100:00100000:2.0:1767319423.428978:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00100000:2.0:1767319423.428986:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982811904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319423.429024:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319423.429068:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982811904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (209us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319423.429076:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00100000:2.0:1767319423.429131:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982811904:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319423.429556:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319423.429741:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319423.429752:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982812032:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319423.429788:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319423.430030:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812032 00000100:00100000:3.0:1767319423.430036:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812032 00000100:00100000:3.0:1767319423.430202:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 753 00000100:00080000:2.0:1767319423.430203:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319423.430213:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982812032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319423.430236:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319423.430326:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982812032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 115us (535us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319423.430333:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 753 00000100:00100000:2.0:1767319423.430387:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982812032:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319424.132587:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319424.964519:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319424.964529:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319424.964585:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319424.964590:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319424.964594:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319424.964601:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4668 00000100:00080000:2.0:1767319424.964606:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319424.964612:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319424.964636:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3800 x1853163982812288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319424.964706:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982812288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319424.964786:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319424.964856:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319424.964866:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982812160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319424.964897:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319424.964929:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812288 00000100:00100000:3.0:1767319424.964936:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812288 00000100:00100000:3.0:1767319424.964967:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812160 00000100:00100000:3.0:1767319424.964979:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812160 00000100:00100000:3.0:1767319424.965043:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106 00000100:00100000:3.0:1767319424.965054:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982812160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319424.965067:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1455 00000100:00100000:1.0:1767319424.965075:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982812288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319424.965096:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319424.965118:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982812160:12345-0@lo:400:kworker.0 Request processed in 65us (219us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319424.965126:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106 00000100:00080000:2.0:1767319424.965171:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319424.965175:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319424.965177:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319424.965180:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4668 00000100:00080000:2.0:1767319424.965183:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319424.965187:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319424.965201:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982812544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319424.965244:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982812416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319424.965268:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767319424.965330:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3100 x1853163982812160/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319440 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319424.965342:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982812160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319424.965368:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812416 00000100:00100000:3.0:1767319424.965372:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812416 00000100:00100000:3.0:1767319424.965395:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107 00000100:00100000:3.0:1767319424.965402:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982812416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319424.965435:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982812416:12345-0@lo:400:kworker.0 Request processed in 34us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319424.965441:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107 00010000:00080000:3.0:1767319424.965463:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3480 x1853163982812416/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319440 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319424.965473:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982812416:0@lo:400:kworker.0 00000100:00080000:3.0:1767319424.965503:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319424.966417:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982812544:0@lo:38:kworker.0 00000100:00100000:0.0:1767319424.966457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319424.967220:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812544 00000100:00100000:3.0:1767319424.967228:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812544 00000100:00100000:3.0:1767319424.967264:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1456 00000100:00100000:3.0:1767319424.967390:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982812544:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319424.967411:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319424.967499:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982812288:12345-0@lo:38:kworker.0 Request processed in 2421us (2711us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319424.967508:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1455 00000100:00080000:0.0:1767319424.967518:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319424.967526:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319424.967530:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982812288:0@lo:38:kworker.0 00000100:00080000:2.0:1767319424.968746:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319424.971650:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319424.971656:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319424.971663:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4668 00000100:00080000:2.0:1767319424.971667:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319424.971675:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319424.971719:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2d80 x1853163982812672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319424.971795:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319424.971799:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319424.971801:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319424.971805:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4668 00000100:00080000:2.0:1767319424.971807:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319424.971811:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319424.971822:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163982813056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319424.971854:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982812928:0@lo:400:kworker.0 00000100:00100000:2.0:1767319424.971916:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319424.971960:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982812800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319424.971982:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812928 00000100:00100000:2.0:1767319424.971990:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812928 00000100:00100000:2.0:1767319424.972050:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 701 00000100:00100000:2.0:1767319424.972058:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982812928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319424.972089:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319424.972141:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812800 00000100:00100000:1.0:1767319424.972145:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812800 00000100:00100000:1.0:1767319424.972170:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767319424.972176:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982812800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319424.972240:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982812800:12345-0@lo:400:kworker.0 Request processed in 63us (151us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319424.972247:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767319424.972322:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982812800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319424.972330:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982812928:12345-0@lo:400:kworker.0 Request processed in 269us (411us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319424.972339:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 701 00000100:00100000:2.0:1767319424.972372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982812928:0@lo:400:kworker.0 00000100:00100000:0.0:1767319424.972776:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982812672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319424.972811:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319424.972861:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982813056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319424.972888:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319424.973019:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813056 00000100:00100000:2.0:1767319424.973025:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813056 00000100:00100000:1.0:1767319424.973111:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982812672 00000100:00100000:1.0:1767319424.973117:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982812672 00000100:00100000:1.0:1767319424.973141:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 389 00000100:00100000:1.0:1767319424.973147:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982812672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319424.973165:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319424.973199:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982812672:12345-0@lo:38:kworker.0 Request processed in 51us (387us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319424.973217:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 389 00000100:00100000:2.0:1767319424.973246:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1457 00000100:00100000:2.0:1767319424.973254:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982813056:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767319424.973265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:2.0:1767319424.973266:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319424.973365:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982813056:12345-0@lo:38:kworker.0 Request processed in 111us (475us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767319424.974146:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319424.974151:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982812672:0@lo:38:kworker.0 00000100:00080000:1.0:1767319424.974172:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319424.974176:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319424.974179:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982813056:0@lo:38:kworker.0 00000100:00100000:2.0:1767319424.976825:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1457 00000100:00100000:3.0:1767319424.977448:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982812544:12345-0@lo:38:kworker.0 Request processed in 10172us (10985us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319424.977459:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1456 00000100:00080000:1.0:1767319424.977582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319424.977589:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319424.977593:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982812544:0@lo:38:kworker.0 00010000:00010000:1.0:1767319425.156596:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319426.180707:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319426.959738:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319426.959778:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624320 00000100:00100000:3.0:1767319426.959785:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319426.959795:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624320 00000100:00080000:3.0:1767319426.959835:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319426.959968:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624192 00000100:00100000:2.0:1767319426.959974:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624192 00000100:00100000:2.0:1767319426.960007:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1108 00000100:00100000:2.0:1767319426.960018:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915624320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319426.960111:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915624320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (398us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319426.960119:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1108 00000100:00100000:2.0:1767319426.960137:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1458 00000100:00100000:2.0:1767319426.960151:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915624192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319426.960171:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319426.960204:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915624192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (418us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319426.960210:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1458 00010000:00010000:1.0:1767319427.204507:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319428.229085:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319428.548710:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319428.551536:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982813312:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319428.551623:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319428.551705:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982813184:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319428.551731:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319428.553137:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813312 00000100:00100000:3.0:1767319428.553150:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813312 00000100:00100000:3.0:1767319428.553220:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813184 00000100:00100000:3.0:1767319428.553224:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813184 00000100:00100000:3.0:1767319428.553245:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 754 00000100:00100000:3.0:1767319428.553255:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982813312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319428.553342:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319428.553407:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982813312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 154us (1784us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319428.553414:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 754 00000100:00100000:3.0:1767319428.553434:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 755 00000100:00100000:3.0:1767319428.553441:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982813184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319428.553457:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319428.553486:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982813184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (1753us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319428.553492:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 755 00000100:00080000:0.0:1767319428.554629:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319428.554638:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319428.554697:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982813312:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319428.554717:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982813184:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319429.252903:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319430.084439:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319430.084450:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319430.084524:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319430.084529:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319430.084533:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319430.084543:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4673 00000100:00080000:2.0:1767319430.084547:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319430.084555:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319430.084576:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163982813568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319430.084612:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319430.084615:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319430.084617:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319430.084620:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4673 00000100:00080000:2.0:1767319430.084622:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319430.084626:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319430.084635:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0e00 x1853163982813824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319430.084642:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319430.084645:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319430.084647:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319430.084650:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4673 00000100:00080000:2.0:1767319430.084653:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767319430.084656:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319430.084666:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163982813952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319430.084694:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319430.084698:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319430.084700:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319430.084703:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4673 00000100:00080000:2.0:1767319430.084705:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767319430.084708:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319430.084719:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163982814336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319430.084739:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982813568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319430.084762:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982814208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319430.084843:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982813440:0@lo:400:kworker.0 00000100:00100000:3.0:1767319430.084843:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982813696:0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.087994:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319430.088100:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319430.088113:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982814080:0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.088157:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319430.088203:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814208 00000100:00100000:2.0:1767319430.088210:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814208 00000100:00100000:2.0:1767319430.088289:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814080 00000100:00100000:2.0:1767319430.088293:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814080 00000100:00100000:2.0:1767319430.088325:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 703 00000100:00100000:2.0:1767319430.088335:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982814208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.088407:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982814208:12345-0@lo:400:kworker.0 Request processed in 73us (421us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319430.088415:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 703 00000100:00100000:2.0:1767319430.088430:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00100000:2.0:1767319430.088436:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982814080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.088462:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982814080:12345-0@lo:400:kworker.0 Request processed in 27us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319430.088467:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00000100:00080000:2.0:1767319430.088486:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319430.088490:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319430.088517:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982814080:0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.088528:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982814208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319430.090503:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319430.090581:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982813824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319430.090631:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319430.090660:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813824 00000100:00100000:2.0:1767319430.090661:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813568 00000100:00100000:2.0:1767319430.090669:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813568 00000100:00100000:2.0:1767319430.090710:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1459 00000100:00100000:2.0:1767319430.090717:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982813568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319430.090739:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319430.090784:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982813568:12345-0@lo:38:kworker.0 Request processed in 66us (287us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319430.090790:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1459 00000100:00100000:1.0:1767319430.093595:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.093712:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982813952:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.093763:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319430.093797:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813952 00000100:00100000:1.0:1767319430.093804:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813952 00000100:00100000:1.0:1767319430.093838:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 390 00000100:00100000:1.0:1767319430.093845:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982813952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319430.093868:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319430.093914:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982813952:12345-0@lo:38:kworker.0 Request processed in 68us (149us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319430.093924:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 390 00000100:00080000:1.0:1767319430.093971:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.093977:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.093980:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982813952:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.093987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982814336:0@lo:38:kworker.0 00000100:00100000:1.0:1767319430.094009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319430.094059:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.094063:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.094066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982813568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319430.094242:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813440 00000100:00100000:2.0:1767319430.094247:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813440 00000100:00100000:2.0:1767319430.094317:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814336 00000100:00100000:2.0:1767319430.094321:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814336 00000100:00100000:2.0:1767319430.094344:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109 00000100:00100000:2.0:1767319430.094351:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982813440:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319430.094410:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982813440:12345-0@lo:400:kworker.0 Request processed in 60us (825us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319430.094418:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109 00000100:00100000:2.0:1767319430.094433:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1460 00000100:00100000:2.0:1767319430.094437:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982814336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319430.094447:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319430.094474:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982814336:12345-0@lo:38:kworker.0 Request processed in 35us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319430.094484:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1460 00000100:00080000:1.0:1767319430.094624:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319430.094652:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1500 x1853163982813440/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319446 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319430.094665:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982813440:0@lo:400:kworker.0 00000100:00080000:1.0:1767319430.094689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.094693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.094696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982814336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319430.095439:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813824 00000100:00100000:0.0:1767319430.095485:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1461 00000100:00100000:0.0:1767319430.095491:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982813824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319430.095506:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319430.095574:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982813824:12345-0@lo:38:kworker.0 Request processed in 82us (4942us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319430.095581:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1461 00000100:00100000:3.0:1767319430.095697:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319430.095939:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319430.095944:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319430.095948:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982813824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319430.096165:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982813696 00000100:00100000:0.0:1767319430.096171:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982813696 00000100:00080000:1.0:1767319430.096406:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319430.096447:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110 00000100:00100000:0.0:1767319430.096455:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982813696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319430.096524:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982813696:12345-0@lo:400:kworker.0 Request processed in 70us (826us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319430.096531:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110 00010000:00080000:1.0:1767319430.097069:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3800 x1853163982813696/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319446 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319430.097084:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982813696:0@lo:400:kworker.0 00010000:00010000:1.0:1767319430.277208:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319431.300510:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319432.076025:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319432.076097:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624576 00000100:00100000:2.0:1767319432.076108:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624576 00000100:00100000:2.0:1767319432.076201:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1111 00000100:00100000:2.0:1767319432.076212:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915624576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319432.076327:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915624576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 117us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319432.076336:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1111 00000100:00080000:1.0:1767319432.076434:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319432.076545:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319432.077028:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624448 00000100:00100000:0.0:1767319432.077035:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624448 00000100:00100000:0.0:1767319432.077066:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1462 00000100:00100000:0.0:1767319432.077072:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915624448:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319432.077094:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319432.077176:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915624448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (631us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319432.077183:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1462 00010000:00010000:3.0:1767319432.325498:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319433.348584:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319433.668629:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982814592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319433.668703:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319433.668788:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982814464:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319433.668823:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319433.668893:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814592 00000100:00100000:2.0:1767319433.668902:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814592 00000100:00080000:3.0:1767319433.668969:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319433.668972:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814464 00000100:00100000:2.0:1767319433.668977:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814464 00000100:00100000:3.0:1767319433.669010:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 756 00000100:00100000:3.0:1767319433.669023:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982814592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319433.669057:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319433.669106:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982814592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (403us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319433.673450:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 757 00000100:00100000:3.0:1767319433.673465:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 756 00000100:00100000:2.0:1767319433.673484:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982814464:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319433.673508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767319433.673526:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319433.673566:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982814592:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319433.673609:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982814464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 143us (4795us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319433.673611:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982814464:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319433.673625:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 757 00010000:00010000:1.0:1767319434.378905:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319435.204671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319435.204682:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319435.204744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319435.204748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319435.204752:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319435.204759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4678 00000100:00080000:1.0:1767319435.204765:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1767319435.204772:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319435.204822:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8e00 x1853163982814848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319435.204862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319435.204866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319435.204868:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319435.204872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4678 00000100:00080000:1.0:1767319435.204874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1767319435.204879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319435.204890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fad80 x1853163982815104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319435.204899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319435.204902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319435.204904:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319435.204907:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4678 00000100:00080000:1.0:1767319435.204910:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1767319435.204913:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319435.204923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163982815232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319435.204957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319435.204961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319435.204962:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319435.204966:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4678 00000100:00080000:1.0:1767319435.204968:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1767319435.204971:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319435.204980:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982815616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319435.205124:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982815488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.205194:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319435.205266:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319435.205304:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982814720:0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.205348:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319435.205385:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814720 00000100:00100000:0.0:1767319435.205401:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814720 00000100:00100000:0.0:1767319435.205458:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112 00000100:00100000:0.0:1767319435.205469:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982814720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319435.205521:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982814720:12345-0@lo:400:kworker.0 Request processed in 54us (171us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319435.205528:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112 00000100:00080000:0.0:1767319435.205548:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319435.205592:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8700 x1853163982814720/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319451 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319435.205606:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982814720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319435.205675:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815488 00000100:00100000:1.0:1767319435.205680:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815488 00000100:00100000:1.0:1767319435.205718:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 705 00000100:00100000:1.0:1767319435.205726:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982815488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319435.205762:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982815488:12345-0@lo:400:kworker.0 Request processed in 38us (568us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319435.205768:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 705 00000100:00080000:0.0:1767319435.206043:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319435.206066:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982815488:0@lo:400:kworker.0 00000100:00100000:3.0:1767319435.206460:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982815360:0@lo:400:kworker.0 00000100:00100000:3.0:1767319435.206504:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319435.206593:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982814976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319435.206625:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319435.206659:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814976 00000100:00100000:2.0:1767319435.206664:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814976 00000100:00100000:2.0:1767319435.206704:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113 00000100:00100000:2.0:1767319435.206710:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982814976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319435.206751:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982814976:12345-0@lo:400:kworker.0 Request processed in 39us (124us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319435.206760:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113 00000100:00080000:3.0:1767319435.206851:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319435.206864:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982814848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319435.206892:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319435.206917:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982815104:0@lo:38:kworker.0 00000100:00100000:3.0:1767319435.206952:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319435.206974:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982815232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319435.206998:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319435.207021:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982815616:0@lo:38:kworker.0 00000100:00100000:3.0:1767319435.207044:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319435.207072:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982814848 00000100:00100000:3.0:1767319435.207076:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982814848 00000100:00100000:3.0:1767319435.207089:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815104 00000100:00100000:3.0:1767319435.207091:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815104 00000100:00100000:3.0:1767319435.207098:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815616 00000100:00100000:3.0:1767319435.207100:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815616 00000100:00100000:3.0:1767319435.207110:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1463 00000100:00100000:3.0:1767319435.207125:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982814848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.207143:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.207178:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982814848:12345-0@lo:38:kworker.0 Request processed in 51us (284us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.207185:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1463 00000100:00100000:3.0:1767319435.207199:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1464 00000100:00100000:3.0:1767319435.207203:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982815104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.207211:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.207234:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982815104:12345-0@lo:38:kworker.0 Request processed in 30us (295us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.207239:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1464 00000100:00100000:3.0:1767319435.207250:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1465 00000100:00100000:3.0:1767319435.207254:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982815616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.207262:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.207310:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982815616:12345-0@lo:38:kworker.0 Request processed in 54us (265us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.207316:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1465 00000100:00100000:3.0:1767319435.207334:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815232 00000100:00100000:3.0:1767319435.207338:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815232 00000100:00100000:3.0:1767319435.207357:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 391 00000100:00100000:3.0:1767319435.207362:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982815232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319435.207371:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319435.207395:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982815232:12345-0@lo:38:kworker.0 Request processed in 32us (396us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319435.207401:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 391 00000100:00080000:3.0:1767319435.207438:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319435.207443:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319435.207446:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982814848:0@lo:38:kworker.0 00000100:00080000:3.0:1767319435.207455:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319435.207458:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319435.207461:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982815104:0@lo:38:kworker.0 00000100:00080000:3.0:1767319435.207469:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319435.207472:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319435.207475:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982815232:0@lo:38:kworker.0 00000100:00080000:3.0:1767319435.207482:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319435.207484:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319435.207488:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982815616:0@lo:38:kworker.0 00010000:00080000:3.0:1767319435.207538:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb480 x1853163982814976/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319451 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319435.207549:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982814976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319435.209061:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815360 00000100:00100000:1.0:1767319435.209074:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815360 00000100:00100000:1.0:1767319435.209166:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767319435.209176:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982815360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319435.209262:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982815360:12345-0@lo:400:kworker.0 Request processed in 89us (2757us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319435.209286:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767319435.209332:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982815360:0@lo:400:kworker.0 00000100:00080000:3.0:1767319435.211081:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767319435.397056:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319436.177418:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319436.177465:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915624704 offset 0 length 520: 4 00000100:00100000:3.0:1767319436.177506:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319436.177956:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624960 00000100:00100000:2.0:1767319436.177977:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624960 00000100:00100000:2.0:1767319436.178045:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915624832 00000100:00100000:2.0:1767319436.178048:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915624832 00000100:00100000:2.0:1767319436.178067:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1114 00000100:00100000:2.0:1767319436.178075:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915624960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319436.178160:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915624960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (754us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319436.178168:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1114 00000100:00100000:2.0:1767319436.178185:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1466 00000100:00100000:2.0:1767319436.178189:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915624832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319436.178221:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319436.178244:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915624832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (739us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319436.178250:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1466 00000100:00080000:3.0:1767319436.178443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319436.420441:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319437.444550:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319438.468573:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319438.788863:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319438.788898:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982815744:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319438.788995:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319438.789107:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319438.789116:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982815872:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319438.789143:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319438.790261:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815744 00000100:00100000:3.0:1767319438.790291:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815744 00000100:00100000:3.0:1767319438.790372:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982815872 00000100:00100000:3.0:1767319438.790376:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982815872 00000100:00100000:3.0:1767319438.790400:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 758 00000100:00100000:3.0:1767319438.790410:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934602e4d850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982815744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319438.790454:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319438.790527:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934602e4d850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982815744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (1531us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319438.790535:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 758 00000100:00100000:3.0:1767319438.790879:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 759 00000100:00100000:3.0:1767319438.790887:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982815872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319438.790903:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319438.790955:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982815872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (1809us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319438.790965:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 759 00000100:00100000:3.0:1767319438.791024:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982815872:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319438.791210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319438.791216:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319438.791248:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982815744:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319439.300934:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319414/real 1767319414] req@ffff9346059cc380 x1853163982809472/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319439 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319439.300987:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346059cc380 x1853163982809472/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319439 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319439.301017:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319439.301046:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319439.301051:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982809472:0@lo:250:kworker.0 00000100:00080000:1.0:1767319439.301241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319439.301248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319439.301252:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319439.301257:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4663 00000100:00080000:1.0:1767319439.301262:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319439.301294:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319439.301340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982816000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319439.301458:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982816128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319439.301560:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319439.301680:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982816000:0@lo:250:kworker.0 00000400:00000100:3.0:1767319439.301727:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982816000 offset 0 length 520: 4 00000100:00100000:3.0:1767319439.301793:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816128 00000100:00100000:3.0:1767319439.301802:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816128 00000100:00100000:3.0:1767319439.301875:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115 00000100:00100000:3.0:1767319439.301887:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982816128:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319439.301961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319439.301966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319439.301968:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319439.301972:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4683 00000100:00080000:1.0:1767319439.301976:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1767319439.301981:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319439.302001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163982816256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319439.302036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319439.302040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319439.302054:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319439.302058:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4683 00000100:00080000:1.0:1767319439.302062:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1767319439.302066:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319439.302078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa300 x1853163982816512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319439.302087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319439.302089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319439.302091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319439.302094:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4683 00000100:00080000:1.0:1767319439.302096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767319439.302100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319439.302111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb800 x1853163982816640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319439.302120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319439.302123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319439.302125:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319439.302129:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4683 00000100:00080000:1.0:1767319439.302132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1767319439.302135:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319439.302145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fbb80 x1853163982816768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319439.302184:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319439.302200:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982816384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319439.302234:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319439.302264:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816384 00000100:00100000:1.0:1767319439.302268:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816384 00000100:00100000:1.0:1767319439.302335:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116 00000100:00100000:1.0:1767319439.302342:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982816384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319439.302380:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982816384:12345-0@lo:400:kworker.0 Request processed in 38us (144us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319439.302387:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116 00000100:00080000:1.0:1767319439.302409:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319439.302437:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8380 x1853163982816384/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319455 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319439.302450:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982816384:0@lo:400:kworker.0 00000100:00100000:1.0:1767319439.302485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982816256:0@lo:38:kworker.0 00000100:00100000:1.0:1767319439.302510:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319439.302527:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982816512:0@lo:38:kworker.0 00000100:00100000:1.0:1767319439.302549:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319439.302565:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982816640:0@lo:38:kworker.0 00000100:00100000:1.0:1767319439.302591:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319439.302610:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982816768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319439.302631:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319439.302657:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816256 00000100:00100000:1.0:1767319439.302661:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816256 00000100:00100000:1.0:1767319439.302672:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816512 00000100:00100000:1.0:1767319439.302675:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816512 00000100:00100000:1.0:1767319439.302682:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816768 00000100:00100000:1.0:1767319439.302684:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816768 00000100:00100000:1.0:1767319439.302693:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1467 00000100:00100000:1.0:1767319439.302697:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319439.302729:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319439.302766:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816256:12345-0@lo:38:kworker.0 Request processed in 66us (255us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319439.302772:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1467 00000100:00100000:1.0:1767319439.302784:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1468 00000100:00100000:1.0:1767319439.302789:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319439.302799:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319439.302822:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816512:12345-0@lo:38:kworker.0 Request processed in 33us (273us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319439.302828:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1468 00000100:00100000:1.0:1767319439.302838:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1469 00000100:00100000:1.0:1767319439.302842:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319439.302850:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319439.302871:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982816768:12345-0@lo:38:kworker.0 Request processed in 28us (239us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319439.302877:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1469 00000100:00080000:1.0:1767319439.302901:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319439.302906:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319439.302909:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982816256:0@lo:38:kworker.0 00000100:00080000:1.0:1767319439.302919:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319439.302923:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319439.302926:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982816512:0@lo:38:kworker.0 00000100:00080000:1.0:1767319439.302934:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319439.302937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319439.302939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982816768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319439.302992:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816640 00000100:00100000:1.0:1767319439.302997:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816640 00000100:00100000:1.0:1767319439.303019:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 392 00000100:00100000:1.0:1767319439.303024:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982816640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319439.303034:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319439.303064:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982816640:12345-0@lo:38:kworker.0 Request processed in 39us (472us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319439.303070:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 392 00000100:00080000:1.0:1767319439.303090:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319439.303094:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319439.303097:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982816640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319439.304954:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982816128:12345-0@lo:400:kworker.0 Request processed in 2789us (3113us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319439.304972:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115 00010000:00080000:0.0:1767319439.305245:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9880 x1853163982816128/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319455 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319439.305263:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982816128:0@lo:400:kworker.0 00010000:00010000:1.0:1767319439.493677:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319440.518600:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319441.292875:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319441.293171:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625088 00000100:00100000:1.0:1767319441.293182:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625088 00000100:00100000:1.0:1767319441.293235:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1470 00000100:00100000:1.0:1767319441.293244:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915625088:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319441.293268:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319441.293372:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915625088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 127us (507us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319441.293380:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1470 00000100:00100000:2.0:1767319441.303359:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319441.304946:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625216 00000100:00100000:0.0:1767319441.304973:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625216 00000100:00100000:0.0:1767319441.305116:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1117 00000100:00100000:0.0:1767319441.305132:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915625216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319441.305325:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915625216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 141us (1917us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319441.305336:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1117 00000100:00080000:1.0:1767319441.305879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767319441.540578:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319442.564915:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319443.588720:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319443.910745:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982816896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319443.910835:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319443.910998:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982816896 00000100:00100000:0.0:1767319443.911008:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982816896 00000100:00100000:0.0:1767319443.911091:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 760 00000100:00100000:0.0:1767319443.911103:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982816896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319443.911139:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319443.911184:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982816896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319443.911191:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 760 00000100:00080000:3.0:1767319443.911866:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319443.911916:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982816896:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319443.912225:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319443.912237:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982817024:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319443.912308:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319443.912391:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817024 00000100:00100000:2.0:1767319443.912397:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817024 00000100:00100000:2.0:1767319443.912462:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 761 00000100:00100000:2.0:1767319443.912471:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982817024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319443.912492:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319443.912535:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982817024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (227us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319443.912542:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 761 00000100:00100000:2.0:1767319443.912574:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982817024:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319443.912636:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319444.426455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319444.426467:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319444.426540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319444.426544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319444.426548:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319444.426557:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4687 00000100:00080000:3.0:1767319444.426563:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319444.426570:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319444.426592:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982817280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319444.426648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319444.426651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319444.426653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319444.426657:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4687 00000100:00080000:3.0:1767319444.426660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767319444.426663:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319444.426674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6d80 x1853163982817536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319444.426684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319444.426687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319444.426688:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319444.426692:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4687 00000100:00080000:3.0:1767319444.426694:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319444.426697:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319444.426709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5180 x1853163982817664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319444.426718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319444.426721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319444.426723:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319444.426726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4687 00000100:00080000:3.0:1767319444.426728:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319444.426731:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319444.426739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6a00 x1853163982817792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319444.426769:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982817408:0@lo:400:kworker.0 00000100:00100000:3.0:1767319444.426844:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319444.426955:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319444.426965:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982817152:0@lo:400:kworker.0 00000100:00100000:1.0:1767319444.426995:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319444.427140:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817408 00000100:00100000:3.0:1767319444.427148:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817408 00000100:00100000:3.0:1767319444.427215:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817152 00000100:00100000:3.0:1767319444.427218:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817152 00000100:00100000:3.0:1767319444.427240:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1767319444.427247:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982817408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319444.427329:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982817408:12345-0@lo:400:kworker.0 Request processed in 82us (494us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319444.427335:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1767319444.427352:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767319444.427359:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982817152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319444.427388:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982817152:12345-0@lo:400:kworker.0 Request processed in 29us (392us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319444.427396:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119 00010000:00080000:3.0:1767319444.427429:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7800 x1853163982817408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319460 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319444.427441:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982817408:0@lo:400:kworker.0 00000100:00080000:1.0:1767319444.427586:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319444.427624:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5880 x1853163982817152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319460 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319444.427633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982817152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319444.427989:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982817280:0@lo:38:kworker.0 00000100:00100000:0.0:1767319444.428025:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319444.428053:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982817536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319444.428071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319444.428170:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982817664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319444.428195:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319444.428218:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982817792:0@lo:38:kworker.0 00000100:00100000:0.0:1767319444.428239:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319444.428293:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817664 00000100:00100000:3.0:1767319444.428299:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817664 00000100:00100000:3.0:1767319444.428325:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 393 00000100:00100000:3.0:1767319444.428331:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982817664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319444.428349:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319444.428383:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982817664:12345-0@lo:38:kworker.0 Request processed in 51us (187us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319444.428389:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 393 00000100:00080000:0.0:1767319444.429691:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319444.429701:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319444.429705:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982817664:0@lo:38:kworker.0 00000100:00100000:1.0:1767319444.429808:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817280 00000100:00100000:1.0:1767319444.429815:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817280 00000100:00100000:1.0:1767319444.429859:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817536 00000100:00100000:1.0:1767319444.429863:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817536 00000100:00100000:1.0:1767319444.429871:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817792 00000100:00100000:1.0:1767319444.429874:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817792 00000100:00100000:1.0:1767319444.429895:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1471 00000100:00100000:1.0:1767319444.429900:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319444.429917:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319444.429968:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817280:12345-0@lo:38:kworker.0 Request processed in 67us (1942us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319444.429974:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1471 00000100:00100000:1.0:1767319444.429989:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1472 00000100:00100000:1.0:1767319444.429992:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319444.430001:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319444.430022:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817536:12345-0@lo:38:kworker.0 Request processed in 29us (1950us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319444.430027:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1472 00000100:00100000:1.0:1767319444.430036:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1473 00000100:00100000:1.0:1767319444.430039:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319444.430051:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319444.430072:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982817792:12345-0@lo:38:kworker.0 Request processed in 31us (1832us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319444.430077:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1473 00000100:00080000:1.0:1767319444.430121:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319444.430126:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319444.430129:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982817280:0@lo:38:kworker.0 00000100:00080000:1.0:1767319444.430138:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319444.430141:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319444.430143:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982817536:0@lo:38:kworker.0 00000100:00080000:1.0:1767319444.430150:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319444.430153:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319444.430155:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982817792:0@lo:38:kworker.0 00010000:00010000:3.0:1767319444.616552:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319445.639408:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319446.412341:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319446.412631:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625472 00000100:00100000:1.0:1767319446.412648:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625472 00000100:00100000:1.0:1767319446.412748:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1120 00000100:00100000:1.0:1767319446.412759:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319446.412860:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (589us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319446.412867:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1120 00000100:00080000:3.0:1767319446.412991:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319446.413119:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319446.414095:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625344 00000100:00100000:1.0:1767319446.414106:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625344 00000100:00100000:1.0:1767319446.414151:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1474 00000100:00100000:1.0:1767319446.414158:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915625344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319446.414509:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319446.414595:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915625344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 434us (1476us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319446.414605:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1474 00010000:00010000:1.0:1767319446.660415:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319447.684503:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319448.708499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319449.028637:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319449.028656:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982817920:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319449.028747:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319449.029922:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982818048:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319449.029964:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319449.037427:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982817920 00000100:00100000:1.0:1767319449.037444:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982817920 00000100:00100000:1.0:1767319449.038373:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818048 00000100:00100000:1.0:1767319449.038382:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818048 00000100:00100000:2.0:1767319449.038451:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 762 00000100:00100000:2.0:1767319449.038464:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982817920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319449.038498:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319449.038594:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982817920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 132us (9846us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319449.038602:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 762 00000100:00100000:2.0:1767319449.038659:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982817920:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319449.038887:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319449.038893:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319449.042033:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 763 00000100:00100000:0.0:1767319449.042053:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982818048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319449.042089:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319449.042178:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982818048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 134us (12212us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319449.042185:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 763 00000100:00100000:1.0:1767319449.043643:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982818048:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319449.540551:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319449.540561:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319449.540631:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319449.540635:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319449.540639:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319449.540648:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4692 00000100:00080000:2.0:1767319449.540653:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319449.540660:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319449.540682:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982818304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319449.540793:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982818176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.540862:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319449.540995:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319449.540999:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319449.541001:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319449.541006:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4692 00000100:00080000:2.0:1767319449.541008:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319449.541013:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319449.541026:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d880 x1853163982818560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319449.541036:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319449.541039:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319449.541041:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319449.541043:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4692 00000100:00080000:2.0:1767319449.541045:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319449.541049:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319449.541063:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163982818688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319449.541087:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319449.541091:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319449.541092:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319449.541096:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4692 00000100:00080000:2.0:1767319449.541098:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319449.541101:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319449.541111:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0380 x1853163982819072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319449.541179:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982818432:0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541209:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319449.541242:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319449.541248:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982818816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541307:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319449.541343:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818816 00000100:00100000:3.0:1767319449.541351:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818816 00000100:00100000:3.0:1767319449.541399:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00100000:3.0:1767319449.541408:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982818816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541459:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982818816:12345-0@lo:400:kworker.0 Request processed in 52us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319449.541467:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00100000:3.0:1767319449.541508:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982818816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541548:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319449.541555:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982818944:0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541578:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319449.541600:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818944 00000100:00100000:3.0:1767319449.541603:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818944 00000100:00100000:3.0:1767319449.541624:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00100000:3.0:1767319449.541630:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982818944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319449.541664:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982818944:12345-0@lo:400:kworker.0 Request processed in 33us (84us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319449.541669:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00100000:3.0:1767319449.541692:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982818944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319449.541874:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818176 00000100:00100000:2.0:1767319449.541879:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818176 00000100:00100000:2.0:1767319449.541925:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818432 00000100:00100000:2.0:1767319449.541929:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818432 00000100:00100000:2.0:1767319449.541950:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121 00000100:00100000:2.0:1767319449.541957:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982818176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319449.541998:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982818176:12345-0@lo:400:kworker.0 Request processed in 41us (1135us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319449.542004:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121 00000100:00100000:2.0:1767319449.542016:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122 00000100:00100000:2.0:1767319449.542021:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982818432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319449.542052:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982818432:12345-0@lo:400:kworker.0 Request processed in 31us (843us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319449.542058:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122 00010000:00080000:2.0:1767319449.542088:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f100 x1853163982818432/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319465 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319449.542098:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982818432:0@lo:400:kworker.0 00000100:00080000:3.0:1767319449.542160:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319449.542182:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543c000 x1853163982818176/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319465 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319449.542191:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982818176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319449.542857:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982818304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319449.543054:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319449.543088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982818560:0@lo:38:kworker.0 00000100:00100000:0.0:1767319449.543110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319449.543206:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982818688:0@lo:38:kworker.0 00000100:00100000:0.0:1767319449.543231:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319449.543297:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982819072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319449.543320:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319449.543368:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818688 00000100:00100000:3.0:1767319449.543375:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818688 00000100:00100000:3.0:1767319449.543403:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 394 00000100:00100000:3.0:1767319449.543410:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982818688:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319449.543428:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319449.543464:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982818688:12345-0@lo:38:kworker.0 Request processed in 54us (231us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319449.543470:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 394 00000100:00080000:0.0:1767319449.543719:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319449.543724:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319449.543728:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982818688:0@lo:38:kworker.0 00000100:00100000:2.0:1767319449.543865:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818304 00000100:00100000:2.0:1767319449.543870:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818304 00000100:00100000:2.0:1767319449.543899:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982818560 00000100:00100000:2.0:1767319449.543910:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982818560 00000100:00100000:2.0:1767319449.543926:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819072 00000100:00100000:2.0:1767319449.543929:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819072 00000100:00100000:2.0:1767319449.543938:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1767319449.543943:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982818304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319449.543955:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319449.543998:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982818304:12345-0@lo:38:kworker.0 Request processed in 53us (1097us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319449.544005:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1767319449.544017:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1476 00000100:00100000:2.0:1767319449.544021:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982818560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319449.544030:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319449.544052:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982818560:12345-0@lo:38:kworker.0 Request processed in 30us (941us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319449.544057:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1476 00000100:00100000:2.0:1767319449.544068:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1477 00000100:00100000:2.0:1767319449.544071:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982819072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319449.544080:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319449.544100:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982819072:12345-0@lo:38:kworker.0 Request processed in 27us (779us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319449.544106:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1477 00000100:00080000:0.0:1767319449.544178:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319449.544183:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319449.544187:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982818304:0@lo:38:kworker.0 00000100:00080000:0.0:1767319449.544195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319449.544199:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319449.544202:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982818560:0@lo:38:kworker.0 00000100:00080000:0.0:1767319449.544210:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319449.544213:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319449.544215:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982819072:0@lo:38:kworker.0 00010000:00010000:1.0:1767319449.732480:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319450.756530:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319451.535414:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319451.535520:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319451.535946:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625728 00000100:00100000:1.0:1767319451.535957:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625728 00000100:00100000:1.0:1767319451.536052:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625600 00000100:00100000:1.0:1767319451.536064:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625600 00000100:00100000:1.0:1767319451.536085:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1123 00000100:00100000:1.0:1767319451.536104:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319451.536200:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 107us (797us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319451.536209:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1123 00000100:00100000:1.0:1767319451.536229:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1478 00000100:00100000:1.0:1767319451.536233:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915625600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319451.536254:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319451.536319:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915625600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 83us (800us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319451.536329:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1478 00000100:00080000:0.0:1767319451.537213:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319451.780613:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319452.805495:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319453.829493:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319454.148490:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319454.148527:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982819200:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319454.148604:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319454.148735:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819200 00000100:00100000:1.0:1767319454.148744:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819200 00000100:00100000:0.0:1767319454.148756:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982819328:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319454.148793:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319454.148821:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819328 00000100:00100000:1.0:1767319454.148825:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819328 00000100:00100000:0.0:1767319454.148841:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 764 00000100:00100000:0.0:1767319454.148852:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982819200:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319454.148878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319454.148884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:0.0:1767319454.148885:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319454.148954:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982819200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319454.149027:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982819200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319454.150021:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 765 00000100:00100000:1.0:1767319454.150037:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982819328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319454.150067:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319454.150191:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982819328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 159us (1396us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319454.150200:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 765 00000100:00100000:0.0:1767319454.150214:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 764 00000100:00100000:3.0:1767319454.151599:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982819328:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319454.661079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319454.661091:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319454.661145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319454.661149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319454.661152:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319454.661158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4698 00000100:00080000:3.0:1767319454.661168:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319454.661174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319454.661190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982819584/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319454.661220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319454.661222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319454.661224:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319454.661226:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4698 00000100:00080000:3.0:1767319454.661227:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319454.661230:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319454.661239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982819840/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319454.661245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319454.661247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319454.661248:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319454.661250:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4698 00000100:00080000:3.0:1767319454.661251:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319454.661253:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319454.661258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163982819968/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319454.661289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319454.661291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319454.661292:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319454.661295:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4698 00000100:00080000:3.0:1767319454.661296:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319454.661298:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319454.661304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543fb80 x1853163982820352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319454.661324:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982820096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.661377:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319454.661434:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319454.661441:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982819712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.661465:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319454.661485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982819584:0@lo:38:kworker.0 00000100:00100000:3.0:1767319454.661512:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819712 00000100:00100000:2.0:1767319454.661521:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319454.661521:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819712 00000100:00100000:2.0:1767319454.661553:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982819840:0@lo:38:kworker.0 00000100:00100000:2.0:1767319454.661576:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319454.661613:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982819968:0@lo:38:kworker.0 00000100:00100000:1.0:1767319454.661634:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820096 00000100:00100000:1.0:1767319454.661640:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820096 00000100:00100000:2.0:1767319454.661641:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319454.661673:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982820352:0@lo:38:kworker.0 00000100:00100000:1.0:1767319454.661681:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 709 00000100:00100000:1.0:1767319454.661693:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982820096:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319454.661700:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319454.661753:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982820096:12345-0@lo:400:kworker.0 Request processed in 62us (374us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319454.661761:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 709 00000100:00100000:3.0:1767319454.662031:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819584 00000100:00100000:3.0:1767319454.662036:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819584 00000100:00100000:3.0:1767319454.662058:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819840 00000100:00100000:3.0:1767319454.662062:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819840 00000100:00100000:3.0:1767319454.662073:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124 00000100:00100000:3.0:1767319454.662080:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982819712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.662113:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982819712:12345-0@lo:400:kworker.0 Request processed in 34us (647us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319454.662119:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124 00000100:00100000:3.0:1767319454.662133:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1479 00000100:00100000:3.0:1767319454.662137:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982819584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319454.662155:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319454.662195:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982819584:12345-0@lo:38:kworker.0 Request processed in 56us (674us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319454.662200:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1479 00000100:00100000:3.0:1767319454.662212:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1480 00000100:00100000:3.0:1767319454.662215:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982819840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319454.662223:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319454.662244:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982819840:12345-0@lo:38:kworker.0 Request processed in 28us (669us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319454.662249:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1480 00000100:00100000:3.0:1767319454.662295:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819968 00000100:00100000:3.0:1767319454.662299:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819968 00000100:00100000:3.0:1767319454.662319:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 395 00000100:00100000:3.0:1767319454.662323:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982819968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319454.662333:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319454.662358:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982819968:12345-0@lo:38:kworker.0 Request processed in 34us (718us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319454.662363:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 395 00000100:00080000:3.0:1767319454.662416:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319454.662447:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982820224:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.662475:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319454.662504:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319454.662510:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982819456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.662543:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820224 00000100:00100000:3.0:1767319454.662547:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820224 00000100:00100000:3.0:1767319454.662570:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000100:00100000:3.0:1767319454.662576:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982820224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.662609:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982820224:12345-0@lo:400:kworker.0 Request processed in 33us (134us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319454.662616:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00010000:00080000:3.0:1767319454.662664:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543d880 x1853163982819712/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319470 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319454.662678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982819712:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.662690:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982820096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319454.665714:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820352 00000100:00100000:3.0:1767319454.665730:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820352 00000100:00100000:3.0:1767319454.665912:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1481 00000100:00100000:3.0:1767319454.665921:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982820352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319454.665943:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319454.666022:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982820352:12345-0@lo:38:kworker.0 Request processed in 102us (4322us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319454.666029:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1481 00000100:00080000:2.0:1767319454.666525:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319454.666533:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319454.666537:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982820352:0@lo:38:kworker.0 00000100:00080000:2.0:1767319454.666566:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319454.666570:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319454.666574:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982819584:0@lo:38:kworker.0 00000100:00080000:2.0:1767319454.666584:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319454.666589:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319454.666604:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319454.666646:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982820224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319454.666900:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982819840:0@lo:38:kworker.0 00000100:00080000:2.0:1767319454.666917:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319454.667289:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982819456 00000100:00100000:1.0:1767319454.667297:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982819456 00000100:00100000:1.0:1767319454.667402:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125 00000100:00100000:1.0:1767319454.667410:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982819456:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319454.667476:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319454.667481:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982819968:0@lo:38:kworker.0 00000100:00100000:1.0:1767319454.667511:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982819456:12345-0@lo:400:kworker.0 Request processed in 88us (894us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319454.667519:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125 00010000:00080000:2.0:1767319454.667541:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5180 x1853163982819456/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319470 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319454.667552:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982819456:0@lo:400:kworker.0 00000100:00080000:3.0:1767319454.667620:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319454.853827:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319455.876691:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319456.654887:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319456.654934:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319456.655004:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625984 00000100:00100000:1.0:1767319456.655018:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625984 00000100:00100000:1.0:1767319456.655105:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915625856 00000100:00100000:1.0:1767319456.655123:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915625856 00000100:00100000:0.0:1767319456.655236:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1126 00000100:00100000:0.0:1767319456.655258:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319456.655353:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1482 00000100:00100000:1.0:1767319456.655362:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915625856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319456.655398:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319456.655466:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915625984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 216us (591us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319456.655478:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1126 00000100:00080000:3.0:1767319456.656299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319456.659370:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915625856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3995us (4422us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319456.659384:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1482 00010000:00010000:3.0:1767319456.903670:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319457.925620:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319458.951150:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319459.268600:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319459.268631:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982820480:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319459.268704:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319459.268771:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319459.268788:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982820608:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319459.268837:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319459.268920:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820480 00000100:00100000:2.0:1767319459.268930:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820480 00000100:00100000:3.0:1767319459.268987:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820608 00000100:00100000:2.0:1767319459.268995:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 766 00000100:00100000:3.0:1767319459.269002:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820608 00000100:00100000:2.0:1767319459.269005:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982820480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319459.269042:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319459.269060:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 767 00000100:00100000:3.0:1767319459.269075:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982820608:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319459.269092:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982820480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (390us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319459.269099:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 766 00002000:00000024:3.0:1767319459.269112:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319459.269186:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982820480:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319459.269188:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982820608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (343us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319459.269202:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 767 00000100:00100000:2.0:1767319459.269245:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982820608:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319459.269300:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319459.269303:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319459.780407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319459.780418:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319459.780487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319459.780491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319459.780495:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319459.780503:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4703 00000100:00080000:3.0:1767319459.780509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319459.780517:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319459.780542:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319459.780542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982820864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319459.780578:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982820736:0@lo:400:kworker.0 00000100:00080000:3.0:1767319459.780612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319459.780616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319459.780618:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319459.780623:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4703 00000100:00080000:3.0:1767319459.780626:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319459.780631:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319459.780645:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163982821120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319459.780654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319459.780658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319459.780660:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319459.780665:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4703 00000100:00080000:3.0:1767319459.780667:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319459.780670:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319459.780680:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982821248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319459.780693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319459.780709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319459.780714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319459.780715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319459.780720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4703 00000100:00080000:3.0:1767319459.780723:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319459.780728:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319459.780747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163982821632/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319459.780785:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982821376:0@lo:400:kworker.0 00000100:00100000:0.0:1767319459.780794:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982821504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319459.780882:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820736 00000100:00100000:1.0:1767319459.780891:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820736 00000100:00100000:1.0:1767319459.780967:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1767319459.780978:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982820736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319459.780984:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982820992:0@lo:400:kworker.0 00000100:00100000:0.0:1767319459.781665:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319459.782859:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319459.782948:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821504 00000100:00100000:3.0:1767319459.782958:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821504 00000100:00100000:3.0:1767319459.783008:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821376 00000100:00100000:3.0:1767319459.783011:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821376 00000100:00100000:3.0:1767319459.783045:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000100:00100000:3.0:1767319459.783059:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982821504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319459.783161:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982821504:12345-0@lo:400:kworker.0 Request processed in 108us (1497us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319459.783174:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00000100:00100000:3.0:1767319459.783197:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 712 00000100:00100000:3.0:1767319459.783203:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982821376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319459.783237:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982821376:12345-0@lo:400:kworker.0 Request processed in 34us (384us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319459.783243:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 712 00000100:00100000:3.0:1767319459.783313:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982821376:0@lo:400:kworker.0 00000100:00080000:3.0:1767319459.783371:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319459.783404:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982820864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319459.783449:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319459.783485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982821120:0@lo:38:kworker.0 00000100:00100000:3.0:1767319459.783518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319459.783931:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982821248:0@lo:38:kworker.0 00000100:00100000:3.0:1767319459.783963:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319459.784109:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982821632:0@lo:38:kworker.0 00000100:00100000:2.0:1767319459.784222:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319459.784262:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820864 00000100:00100000:2.0:1767319459.784267:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820864 00000100:00100000:2.0:1767319459.784304:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821120 00000100:00100000:2.0:1767319459.784307:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821120 00000100:00100000:2.0:1767319459.784321:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982820992 00000100:00100000:2.0:1767319459.784323:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982820992 00000100:00100000:2.0:1767319459.784372:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128 00000100:00100000:1.0:1767319459.784374:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982820736:12345-0@lo:400:kworker.0 Request processed in 3389us (3676us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319459.784387:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982820992:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319459.784397:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1767319459.784446:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1483 00000100:00100000:2.0:1767319459.784455:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982820992:12345-0@lo:400:kworker.0 Request processed in 72us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319459.784456:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982820864:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319459.784480:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128 00010000:00020000:1.0:1767319459.784498:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319459.784504:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1484 00000100:00100000:2.0:1767319459.784511:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982821120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319459.784531:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319459.784554:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982820864:12345-0@lo:38:kworker.0 Request processed in 97us (1101us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319459.784562:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1483 00000100:00100000:1.0:1767319459.784658:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821248 00000100:00100000:1.0:1767319459.784666:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821248 00000100:00100000:1.0:1767319459.784697:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 396 00000100:00100000:1.0:1767319459.784703:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982821248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319459.784716:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319459.784725:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982821120:12345-0@lo:38:kworker.0 Request processed in 206us (1204us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319459.784735:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1484 00000100:00100000:1.0:1767319459.784768:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982821248:12345-0@lo:38:kworker.0 Request processed in 61us (801us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319459.784777:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 396 00010000:00080000:2.0:1767319459.784785:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015c00 x1853163982820992/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319475 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319459.784801:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319459.784807:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982820992:0@lo:400:kworker.0 00000100:00100000:3.0:1767319459.784962:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319459.785042:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319459.785048:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319459.785052:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982820864:0@lo:38:kworker.0 00000100:00080000:3.0:1767319459.785059:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319459.785063:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319459.785065:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982821120:0@lo:38:kworker.0 00000100:00080000:3.0:1767319459.785073:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319459.785076:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319459.785078:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982821248:0@lo:38:kworker.0 00000100:00100000:2.0:1767319459.785152:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821632 00000100:00100000:2.0:1767319459.785158:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821632 00000100:00100000:2.0:1767319459.785188:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1485 00000100:00100000:2.0:1767319459.785193:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982821632:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319459.785205:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319459.785251:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982821632:12345-0@lo:38:kworker.0 Request processed in 57us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319459.785258:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1485 00000100:00080000:3.0:1767319459.785363:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319459.785368:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319459.785371:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982821632:0@lo:38:kworker.0 00000100:00100000:0.0:1767319459.785628:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982821504:0@lo:400:kworker.0 00010000:00080000:0.0:1767319459.785643:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543e300 x1853163982820736/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319475 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319459.785650:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982820736:0@lo:400:kworker.0 00010000:00010000:3.0:1767319459.973080:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319460.748239:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915626112 offset 0 length 520: 4 00000100:00100000:3.0:1767319460.748495:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319460.748548:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319460.749097:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626240 00000100:00100000:2.0:1767319460.749115:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626240 00000100:00100000:2.0:1767319460.749181:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626368 00000100:00100000:2.0:1767319460.749185:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626368 00000100:00100000:2.0:1767319460.749259:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1129 00000100:00100000:2.0:1767319460.749332:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915626368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319460.749345:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1486 00000100:00100000:0.0:1767319460.749357:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915626240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319460.749385:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319460.749446:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915626368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 175us (895us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319460.749457:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1129 00000100:00100000:0.0:1767319460.749466:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915626240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 106us (969us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319460.749478:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1486 00000100:00080000:3.0:1767319460.750076:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319460.998644:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319462.020876:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319463.044518:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319464.068445:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319464.389431:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982821760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319464.389497:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.389541:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821760 00000100:00100000:1.0:1767319464.389550:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821760 00000100:00100000:1.0:1767319464.389599:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 768 00000100:00100000:1.0:1767319464.389607:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982821760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319464.389637:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319464.389684:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982821760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319464.389691:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 768 00000100:00100000:0.0:1767319464.389783:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982821760:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319464.389901:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319464.390723:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319464.390735:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982821888:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319464.390774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.390814:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982821888 00000100:00100000:1.0:1767319464.390819:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982821888 00000100:00100000:1.0:1767319464.390851:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 769 00000100:00100000:1.0:1767319464.390855:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982821888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319464.390871:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767319464.390875:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319464.390904:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982821888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (129us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319464.390910:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 769 00000100:00100000:3.0:1767319464.391179:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982821888:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319464.901671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319464.901682:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319464.901699:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319439/real 1767319439] req@ffff9346076f9180 x1853163982816000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319464 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319464.901731:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076f9180 x1853163982816000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319464 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319464.901750:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319464.901751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319464.901757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319464.901760:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319464.901767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4708 00000100:00100000:0.0:1767319464.901772:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982822016:0@lo:400:kworker.0 00000100:00080000:3.0:1767319464.901781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319464.901782:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319464.901789:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982816000:0@lo:250:kworker.0 00000100:00080000:3.0:1767319464.901792:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319464.901817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626300 x1853163982822144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319464.901830:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319464.901852:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982822144:0@lo:38:kworker.0 00000100:00080000:3.0:1767319464.901870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319464.901875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319464.901878:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319464.901882:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4708 00000100:00080000:3.0:1767319464.901885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319464.901892:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319464.901892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319464.901903:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319464.901910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982822400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319464.901912:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982822272:0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.901937:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319464.901940:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822016 00000100:00100000:2.0:1767319464.901948:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822016 00000100:00100000:2.0:1767319464.902002:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822144 00000100:00100000:2.0:1767319464.902005:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822144 00000100:00100000:0.0:1767319464.902117:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822272 00000100:00100000:0.0:1767319464.902121:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822272 00000100:00100000:0.0:1767319464.902151:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130 00000100:00100000:0.0:1767319464.902160:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982822016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.902207:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982822016:12345-0@lo:400:kworker.0 Request processed in 48us (378us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319464.902214:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130 00000100:00100000:0.0:1767319464.902229:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1131 00000100:00100000:0.0:1767319464.902246:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982822272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.902300:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982822272:12345-0@lo:400:kworker.0 Request processed in 65us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319464.902306:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1131 00000100:00100000:0.0:1767319464.902319:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1487 00000100:00100000:0.0:1767319464.902323:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982822144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319464.902353:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319464.905851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319464.905883:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982822144:12345-0@lo:38:kworker.0 Request processed in 3552us (3987us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319464.905896:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1487 00010000:00080000:0.0:1767319464.906071:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543e300 x1853163982822272/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319480 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319464.906087:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982822272:0@lo:400:kworker.0 00000100:00080000:3.0:1767319464.906526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319464.906531:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00010000:00080000:0.0:1767319464.906898:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626a00 x1853163982822016/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319480 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319464.906909:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982822016:0@lo:400:kworker.0 00000100:00080000:0.0:1767319464.907063:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319464.907087:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982822400:0@lo:38:kworker.0 00000100:00100000:0.0:1767319464.907144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319464.907188:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319464.907192:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319464.907196:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982822144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319464.907220:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822400 00000100:00100000:0.0:1767319464.907226:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822400 00000100:00100000:1.0:1767319464.907496:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1488 00000100:00100000:1.0:1767319464.907506:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982822400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319464.907527:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319464.907588:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982822400:12345-0@lo:38:kworker.0 Request processed in 82us (443us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319464.907595:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1488 00000100:00080000:0.0:1767319464.907821:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319464.907827:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319464.907830:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982822400:0@lo:38:kworker.0 00000100:00080000:3.0:1767319464.908629:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4708 00000100:00080000:3.0:1767319464.908640:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319464.908645:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319464.908674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d500 x1853163982822528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319464.908731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319464.908734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319464.908736:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319464.908738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4708 00000100:00080000:3.0:1767319464.908740:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319464.908743:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319464.908750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982822912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319464.908763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319464.908765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319464.908766:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319464.908776:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4687 00000100:00080000:3.0:1767319464.908778:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319464.908780:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319464.908786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163982823040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319464.908805:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982822784:0@lo:400:kworker.0 00000100:00100000:3.0:1767319464.908837:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.908964:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822784 00000100:00100000:1.0:1767319464.908969:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822784 00000100:00100000:1.0:1767319464.909008:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 713 00000100:00100000:1.0:1767319464.909013:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982822784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319464.909047:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982822784:12345-0@lo:400:kworker.0 Request processed in 35us (209us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319464.909052:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 713 00000100:00100000:1.0:1767319464.909068:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982822656:0@lo:400:kworker.0 00000100:00100000:1.0:1767319464.909101:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.909132:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822656 00000100:00100000:0.0:1767319464.909136:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982822528:0@lo:38:kworker.0 00000100:00100000:1.0:1767319464.909137:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822656 00000100:00100000:1.0:1767319464.909164:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000100:00100000:0.0:1767319464.909175:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.909184:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982822656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.909208:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982822912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319464.909230:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982822656:12345-0@lo:400:kworker.0 Request processed in 48us (127us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319464.909250:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319464.909257:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00100000:0.0:1767319464.909294:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982823040:0@lo:250:kworker.0 00000400:00000100:0.0:1767319464.909313:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982823040 offset 0 length 520: 4 00000100:00100000:1.0:1767319464.909714:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822528 00000100:00100000:1.0:1767319464.909718:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822528 00000100:00100000:3.0:1767319464.909777:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982822784:0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.911924:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982822656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319464.911963:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982822912 00000100:00100000:0.0:1767319464.911968:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982822912 00000100:00100000:0.0:1767319464.912011:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1489 00000100:00100000:0.0:1767319464.912018:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982822912:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319464.912109:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319464.912178:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982822912:12345-0@lo:38:kworker.0 Request processed in 159us (2930us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319464.912189:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1489 00000100:00080000:0.0:1767319464.912217:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319464.912222:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319464.912225:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982822912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319464.912736:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 397 00000100:00100000:1.0:1767319464.912745:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982822528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319464.912760:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319464.912809:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982822528:12345-0@lo:38:kworker.0 Request processed in 64us (3634us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319464.912815:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 397 00000100:00080000:0.0:1767319464.912848:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319464.912854:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319464.912857:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982822528:0@lo:38:kworker.0 00010000:00010000:1.0:1767319465.092671:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319465.868357:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319465.868430:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319465.868522:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626496 00000100:00100000:0.0:1767319465.868529:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626496 00000100:00100000:1.0:1767319465.868556:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626624 00000100:00100000:1.0:1767319465.868571:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626624 00000100:00100000:0.0:1767319465.868574:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1490 00000100:00100000:0.0:1767319465.868584:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915626496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319465.868620:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319465.868654:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319465.868683:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1132 00000100:00100000:1.0:1767319465.868696:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915626624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319465.868805:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915626624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 110us (372us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319465.869143:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915626496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 556us (792us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319465.869154:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1490 00000100:00100000:1.0:1767319465.872651:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1132 00010000:00010000:3.0:1767319466.116756:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319467.142335:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319468.164541:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319469.188490:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319469.508544:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319469.508616:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982823168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319469.508689:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319469.508934:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982823296:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319469.508962:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319469.509505:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823168 00000100:00100000:1.0:1767319469.509514:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823168 00000100:00080000:3.0:1767319469.509608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319469.509645:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823296 00000100:00100000:2.0:1767319469.509649:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823296 00000100:00100000:2.0:1767319469.509735:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 770 00000100:00100000:2.0:1767319469.509744:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982823168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319469.509775:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319469.509828:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982823168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (1138us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319469.509835:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 770 00000100:00100000:2.0:1767319469.509854:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 771 00000100:00100000:2.0:1767319469.509860:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982823296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319469.509874:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319469.509902:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982823296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (939us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319469.509907:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 771 00000100:00100000:2.0:1767319469.509946:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982823296:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319469.509977:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982823168:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319469.510100:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319470.021035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319470.021044:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319470.021099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319470.021103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319470.021106:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319470.021114:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4713 00000100:00080000:3.0:1767319470.021118:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319470.021124:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319470.021154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163982823552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319470.021187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319470.021190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319470.021192:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319470.021195:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4713 00000100:00080000:3.0:1767319470.021197:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319470.021201:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319470.021211:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7100 x1853163982823808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319470.021219:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319470.021222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319470.021224:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319470.021227:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4713 00000100:00080000:3.0:1767319470.021229:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319470.021232:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319470.021242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6d80 x1853163982823936/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319470.021251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319470.021255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319470.021256:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319470.021260:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4713 00000100:00080000:3.0:1767319470.021263:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319470.021266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319470.021297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163982824064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319470.021323:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319470.021335:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982823680:0@lo:400:kworker.0 00000100:00100000:3.0:1767319470.021401:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319470.021439:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982823552:0@lo:38:kworker.0 00000100:00100000:3.0:1767319470.021460:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319470.021475:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982823808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319470.021493:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319470.021507:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982823936:0@lo:38:kworker.0 00000100:00100000:3.0:1767319470.021529:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319470.021547:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982824064:0@lo:38:kworker.0 00000100:00100000:3.0:1767319470.021565:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319470.021595:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823936 00000100:00100000:3.0:1767319470.021602:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823936 00000100:00100000:3.0:1767319470.021638:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 398 00000100:00100000:3.0:1767319470.021645:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982823936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319470.021664:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319470.021701:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982823936:12345-0@lo:38:kworker.0 Request processed in 56us (171us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319470.021708:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 398 00000100:00080000:3.0:1767319470.021747:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319470.021750:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319470.021753:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982823936:0@lo:38:kworker.0 00000100:00100000:1.0:1767319470.021820:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823680 00000100:00100000:1.0:1767319470.021823:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823680 00000100:00100000:1.0:1767319470.021858:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823552 00000100:00100000:1.0:1767319470.021860:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823552 00000100:00100000:1.0:1767319470.021865:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823808 00000100:00100000:1.0:1767319470.021866:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823808 00000100:00100000:1.0:1767319470.021871:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824064 00000100:00100000:1.0:1767319470.021873:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824064 00000100:00100000:1.0:1767319470.021879:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1133 00000100:00100000:1.0:1767319470.021884:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982823680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319470.021920:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982823680:12345-0@lo:400:kworker.0 Request processed in 38us (520us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319470.021924:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1133 00000100:00100000:1.0:1767319470.021933:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1491 00000100:00100000:1.0:1767319470.021936:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982823552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319470.021942:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319470.021958:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982823552:12345-0@lo:38:kworker.0 Request processed in 21us (498us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319470.021962:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1491 00000100:00100000:1.0:1767319470.021968:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1492 00000100:00100000:1.0:1767319470.021970:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982823808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319470.021975:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319470.021986:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982823808:12345-0@lo:38:kworker.0 Request processed in 15us (492us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319470.021989:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1492 00000100:00100000:1.0:1767319470.021994:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1493 00000100:00100000:1.0:1767319470.021996:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982824064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319470.022001:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319470.022010:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982824064:12345-0@lo:38:kworker.0 Request processed in 14us (445us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319470.022014:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1493 00010000:00080000:1.0:1767319470.022029:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015f80 x1853163982823680/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319486 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319470.022036:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982823680:0@lo:400:kworker.0 00000100:00080000:1.0:1767319470.022050:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319470.022053:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319470.022055:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982823552:0@lo:38:kworker.0 00000100:00080000:1.0:1767319470.022059:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319470.022061:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319470.022062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982823808:0@lo:38:kworker.0 00000100:00080000:1.0:1767319470.022066:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319470.022068:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319470.022070:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982824064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319470.022079:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319470.022084:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982823424:0@lo:400:kworker.0 00000100:00100000:1.0:1767319470.022098:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319470.022111:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982823424 00000100:00100000:1.0:1767319470.022113:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982823424 00000100:00100000:1.0:1767319470.022126:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1134 00000100:00100000:1.0:1767319470.022131:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982823424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319470.022174:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982823424:12345-0@lo:400:kworker.0 Request processed in 42us (74us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319470.022180:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1134 00010000:00080000:1.0:1767319470.022202:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014700 x1853163982823424/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319486 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319470.022208:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982823424:0@lo:400:kworker.0 00000100:00080000:3.0:1767319470.022355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319470.213372:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319471.009028:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319471.009147:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319471.009253:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626880 00000100:00100000:0.0:1767319471.009263:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626880 00000100:00100000:0.0:1767319471.009374:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915626752 00000100:00100000:0.0:1767319471.009378:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915626752 00000100:00100000:0.0:1767319471.009401:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1135 00000100:00100000:0.0:1767319471.009412:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915626880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319471.009469:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319471.009544:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1494 00000100:00100000:2.0:1767319471.009552:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915626752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319471.009572:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319471.009600:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915626752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (453us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319471.009606:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1494 00000100:00100000:0.0:1767319471.010013:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915626880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (478us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319471.010020:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1135 00010000:00010000:1.0:1767319471.236738:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319472.261344:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319473.295455:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319474.310509:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319474.628742:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982824192:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319474.628835:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319474.629003:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824192 00000100:00100000:2.0:1767319474.629014:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824192 00000100:00100000:1.0:1767319474.629163:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319474.629174:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982824320:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319474.629219:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319474.629250:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824320 00000100:00100000:1.0:1767319474.629253:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824320 00000100:00100000:2.0:1767319474.629392:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 772 00000100:00100000:2.0:1767319474.629403:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982824192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319474.629441:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319474.629490:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982824192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (654us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319474.629497:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 772 00000100:00100000:2.0:1767319474.629552:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982824192:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319474.630175:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319474.630181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319474.630747:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 773 00000100:00100000:1.0:1767319474.630759:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982824320:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319474.630787:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319474.630854:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982824320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (1634us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319474.630861:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 773 00000100:00100000:2.0:1767319474.630986:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982824320:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319475.140965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319475.140975:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319475.141034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319475.141038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319475.141102:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319475.141111:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4718 00000100:00080000:3.0:1767319475.141116:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319475.141124:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319475.141151:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7100 x1853163982824576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319475.141154:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982824448:0@lo:400:kworker.0 00000100:00080000:3.0:1767319475.141214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319475.141219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319475.141220:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319475.141224:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4718 00000100:00080000:3.0:1767319475.141227:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319475.141231:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319475.141245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962a00 x1853163982824832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319475.141254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319475.141257:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319475.141259:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319475.141303:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4718 00000100:00080000:3.0:1767319475.141305:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319475.141309:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319475.141315:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319475.141325:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982824960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319475.141336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319475.141338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319475.141340:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319475.141343:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4718 00000100:00080000:3.0:1767319475.141345:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319475.141348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319475.141359:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163982825088/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319475.141419:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824448 00000100:00100000:1.0:1767319475.141430:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824448 00000100:00100000:1.0:1767319475.141514:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136 00000100:00100000:1.0:1767319475.141529:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982824448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319475.141552:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319475.141564:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982824704:0@lo:400:kworker.0 00000100:00100000:2.0:1767319475.141599:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319475.141612:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982824448:12345-0@lo:400:kworker.0 Request processed in 85us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319475.141622:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136 00000100:00100000:1.0:1767319475.141644:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824704 00000100:00080000:2.0:1767319475.141644:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319475.141650:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824704 00000100:00100000:2.0:1767319475.141667:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982824576:0@lo:38:kworker.0 00000100:00100000:1.0:1767319475.141683:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137 00000100:00100000:1.0:1767319475.141692:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982824704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319475.141700:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319475.141719:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982824832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319475.141927:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824576 00000100:00100000:0.0:1767319475.141937:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824576 00000100:00100000:0.0:1767319475.141970:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1495 00000100:00100000:0.0:1767319475.141977:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982824576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319475.141996:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319475.142358:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319475.142433:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6d80 x1853163982824704/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319491 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319475.142448:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982824704:0@lo:400:kworker.0 00010000:00080000:3.0:1767319475.142526:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6680 x1853163982824448/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319491 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319475.142534:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982824448:0@lo:400:kworker.0 00000100:00100000:2.0:1767319475.144631:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319475.144694:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982824960:0@lo:38:kworker.0 00000100:00100000:2.0:1767319475.144761:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319475.144815:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982825088:0@lo:38:kworker.0 00000100:00100000:2.0:1767319475.144840:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319475.144891:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824960 00000100:00100000:2.0:1767319475.144899:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824960 00000100:00100000:2.0:1767319475.144938:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 399 00000100:00100000:2.0:1767319475.144946:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982824960:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319475.144971:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319475.145012:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982824960:12345-0@lo:38:kworker.0 Request processed in 66us (249us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319475.145020:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 399 00000100:00100000:2.0:1767319475.145042:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982824832 00000100:00100000:2.0:1767319475.145057:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982824832 00000100:00100000:2.0:1767319475.145075:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825088 00000100:00100000:2.0:1767319475.145078:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825088 00000100:00100000:2.0:1767319475.145088:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1496 00000100:00100000:2.0:1767319475.145092:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982824832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319475.145101:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319475.145137:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982824832:12345-0@lo:38:kworker.0 Request processed in 44us (515us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319475.145143:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1496 00000100:00100000:2.0:1767319475.145153:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1497 00000100:00100000:2.0:1767319475.145157:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982825088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319475.145166:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319475.145186:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982825088:12345-0@lo:38:kworker.0 Request processed in 27us (345us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319475.145191:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1497 00000100:00080000:2.0:1767319475.145244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319475.145249:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319475.145254:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982824832:0@lo:38:kworker.0 00000100:00080000:2.0:1767319475.145262:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319475.145265:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319475.145268:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982824960:0@lo:38:kworker.0 00000100:00080000:2.0:1767319475.145300:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319475.145303:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319475.145306:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982825088:0@lo:38:kworker.0 00000100:00100000:1.0:1767319475.149338:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982824704:12345-0@lo:400:kworker.0 Request processed in 7629us (7720us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319475.149359:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137 00000100:00100000:0.0:1767319475.152213:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982824576:12345-0@lo:38:kworker.0 Request processed in 10225us (10502us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319475.152228:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1495 00000100:00080000:2.0:1767319475.152669:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319475.152682:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319475.152687:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982824576:0@lo:38:kworker.0 00010000:00010000:3.0:1767319475.337603:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319476.107823:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319476.107962:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319476.108251:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627136 00000100:00100000:0.0:1767319476.108260:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627136 00000100:00080000:3.0:1767319476.108352:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319476.108632:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627008 00000100:00100000:1.0:1767319476.108636:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627008 00000100:00100000:1.0:1767319476.108695:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1138 00000100:00100000:1.0:1767319476.108705:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915627136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319476.108884:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1498 00000100:00100000:0.0:1767319476.108891:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915627008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319476.108909:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319476.109036:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915627136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 323us (1214us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319476.109045:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1138 00000100:00100000:0.0:1767319476.109098:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915627008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 207us (1137us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319476.109103:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1498 00010000:00010000:1.0:1767319476.371757:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319477.380613:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319478.404539:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319479.428561:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319479.748618:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319479.748643:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982825216:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319479.748712:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319479.748911:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825216 00000100:00100000:2.0:1767319479.748920:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825216 00000100:00100000:2.0:1767319479.748996:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 774 00000100:00100000:2.0:1767319479.749006:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982825216:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319479.749043:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319479.749091:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982825216:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (378us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319479.749103:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 774 00000100:00080000:0.0:1767319479.750065:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319479.750115:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982825216:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319479.750131:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319479.750141:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982825344:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319479.750181:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319479.750904:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825344 00000100:00100000:0.0:1767319479.750912:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825344 00000100:00100000:0.0:1767319479.751504:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 775 00000100:00100000:0.0:1767319479.751515:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982825344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319479.751551:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319479.751654:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982825344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (1472us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319479.751662:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 775 00000100:00100000:3.0:1767319479.751998:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982825344:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319479.752950:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319480.260754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319480.260763:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319480.260864:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982825472:0@lo:400:kworker.0 00000100:00100000:1.0:1767319480.260932:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319480.261066:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825472 00000100:00100000:0.0:1767319480.261074:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825472 00000100:00100000:0.0:1767319480.261153:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767319480.261163:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982825472:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319480.261243:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319480.261661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319480.261667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319480.261670:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319480.261675:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4723 00000100:00080000:3.0:1767319480.261679:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319480.261685:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319480.261709:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961500 x1853163982825600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319480.261748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319480.261752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319480.261754:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319480.261759:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4723 00000100:00080000:3.0:1767319480.261761:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319480.261765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319480.261777:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963b80 x1853163982825856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319480.261785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319480.261788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319480.261790:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319480.261793:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4723 00000100:00080000:3.0:1767319480.261795:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319480.261799:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319480.261810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163982825984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319480.261819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319480.261823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319480.261824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319480.261827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4723 00000100:00080000:3.0:1767319480.261830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319480.261833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319480.261843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960e00 x1853163982826112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319480.261870:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982825600:0@lo:38:kworker.0 00000100:00100000:3.0:1767319480.261905:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.261935:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982825856:0@lo:38:kworker.0 00000100:00100000:3.0:1767319480.261971:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.261993:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982825984:0@lo:38:kworker.0 00000100:00100000:3.0:1767319480.262019:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.262043:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982826112:0@lo:38:kworker.0 00000100:00100000:3.0:1767319480.262064:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.262091:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319480.262099:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982825728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319480.263034:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319480.263090:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825600 00000100:00100000:3.0:1767319480.263097:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825600 00000100:00100000:2.0:1767319480.263513:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825984 00000100:00100000:2.0:1767319480.263519:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825984 00000100:00100000:2.0:1767319480.263566:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 400 00000100:00100000:2.0:1767319480.263574:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982825984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319480.263595:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319480.263647:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982825984:12345-0@lo:38:kworker.0 Request processed in 72us (1627us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319480.263653:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 400 00000100:00100000:2.0:1767319480.263678:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825856 00000100:00100000:2.0:1767319480.263683:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825856 00000100:00100000:2.0:1767319480.263695:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826112 00000100:00100000:2.0:1767319480.263697:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826112 00000100:00100000:2.0:1767319480.263705:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982825728 00000100:00100000:2.0:1767319480.263707:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982825728 00000100:00100000:2.0:1767319480.263736:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140 00000100:00100000:2.0:1767319480.263743:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982825728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319480.263786:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982825728:12345-0@lo:400:kworker.0 Request processed in 43us (752us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319480.263792:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140 00000100:00100000:2.0:1767319480.263804:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1499 00000100:00100000:2.0:1767319480.263808:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982825856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319480.263817:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319480.263840:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982825856:12345-0@lo:38:kworker.0 Request processed in 31us (1868us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319480.263849:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1499 00000100:00100000:2.0:1767319480.263860:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1500 00000100:00100000:2.0:1767319480.263864:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982826112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319480.263871:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319480.263893:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982826112:12345-0@lo:38:kworker.0 Request processed in 27us (1829us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319480.263898:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1500 00000100:00080000:2.0:1767319480.263912:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319480.263944:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962d80 x1853163982825728/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319496 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319480.263968:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982825728:0@lo:400:kworker.0 00000100:00080000:2.0:1767319480.264004:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.264009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.264012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982825856:0@lo:38:kworker.0 00000100:00080000:2.0:1767319480.264020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.264023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.264025:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982825984:0@lo:38:kworker.0 00000100:00080000:2.0:1767319480.264032:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319480.264035:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319480.264038:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982826112:0@lo:38:kworker.0 00000100:00100000:3.0:1767319480.265068:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1501 00000100:00100000:3.0:1767319480.265075:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982825600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319480.265089:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319480.266749:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982825600:12345-0@lo:38:kworker.0 Request processed in 1666us (4836us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319480.266762:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1501 00000100:00080000:3.0:1767319480.266825:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319480.266829:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319480.266834:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982825600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319480.268885:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982825472:12345-0@lo:400:kworker.0 Request processed in 7714us (7943us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319480.268899:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1139 00010000:00080000:1.0:1767319480.274885:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163982825472/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319496 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319480.274904:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982825472:0@lo:400:kworker.0 00010000:00010000:1.0:1767319480.454127:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319481.229335:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319481.229424:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319481.233403:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627392 00000100:00100000:2.0:1767319481.233417:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627392 00000100:00100000:2.0:1767319481.233507:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627264 00000100:00100000:2.0:1767319481.233511:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627264 00000100:00100000:2.0:1767319481.233538:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1141 00000100:00100000:2.0:1767319481.233548:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915627392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319481.233861:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319481.233893:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1502 00000100:00100000:3.0:1767319481.233900:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915627264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319481.233918:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319481.234069:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915627392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 521us (4743us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319481.234079:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1141 00000100:00100000:3.0:1767319481.238525:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915627264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 4610us (9087us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319481.238542:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1502 00010000:00010000:1.0:1767319481.476539:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319482.502339:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319483.524428:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319484.548593:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319484.868521:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319484.868544:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982826240:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319484.868614:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319484.868727:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826240 00000100:00100000:1.0:1767319484.868735:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826240 00000100:00100000:0.0:1767319484.868742:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319484.868755:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982826368:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319484.868791:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319484.868961:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319484.874510:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826368 00000100:00100000:1.0:1767319484.874524:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826368 00000100:00100000:1.0:1767319484.874609:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 776 00000100:00100000:1.0:1767319484.874620:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982826240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319484.874656:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319484.874729:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982826240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (6116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319484.874735:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 776 00000100:00100000:1.0:1767319484.874748:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 777 00000100:00100000:1.0:1767319484.874752:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982826368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319484.874763:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319484.874784:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982826368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 32us (5995us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319484.874787:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 777 00000100:00080000:0.0:1767319484.874842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319484.874899:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982826368:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319484.879000:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982826240:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319485.382373:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319485.382383:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319485.382490:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982826496:0@lo:400:kworker.0 00000100:00100000:0.0:1767319485.382563:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319485.382676:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826496 00000100:00100000:3.0:1767319485.382684:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826496 00000100:00100000:3.0:1767319485.382746:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1142 00000100:00100000:3.0:1767319485.382757:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982826496:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319485.382855:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319485.383200:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319485.383204:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319485.383208:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319485.383213:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4728 00000100:00080000:2.0:1767319485.383217:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319485.383221:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319485.383244:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163982826624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319485.383311:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319485.383314:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319485.383316:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319485.383320:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4728 00000100:00080000:2.0:1767319485.383323:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319485.383326:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319485.383340:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982826880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319485.383348:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319485.383351:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319485.383353:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319485.383356:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4728 00000100:00080000:2.0:1767319485.383359:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319485.383362:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319485.383372:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1c00 x1853163982827008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319485.383378:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319485.383381:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319485.383382:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319485.383384:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4728 00000100:00080000:2.0:1767319485.383386:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319485.383387:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319485.383392:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982827136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319485.383424:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982826752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.383448:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.383471:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826752 00000100:00100000:2.0:1767319485.383474:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826752 00000100:00100000:2.0:1767319485.383498:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143 00000100:00100000:2.0:1767319485.383504:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982826752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.383539:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982826752:12345-0@lo:400:kworker.0 Request processed in 34us (88us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319485.383546:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143 00000100:00080000:2.0:1767319485.383572:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319485.383591:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c2d80 x1853163982826752/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319501 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319485.383600:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982826752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319485.383628:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982826624:0@lo:38:kworker.0 00000100:00100000:2.0:1767319485.383652:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.383672:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982826880:0@lo:38:kworker.0 00000100:00100000:2.0:1767319485.383684:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.383694:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982827008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319485.383708:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.383720:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982827136:0@lo:38:kworker.0 00000100:00100000:2.0:1767319485.383731:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319485.383748:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827008 00000100:00100000:2.0:1767319485.383751:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827008 00000100:00100000:2.0:1767319485.383763:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 401 00000100:00100000:2.0:1767319485.383766:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982827008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319485.383779:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319485.383812:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982827008:12345-0@lo:38:kworker.0 Request processed in 44us (102us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.383816:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 401 00000100:00100000:2.0:1767319485.383828:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826624 00000100:00100000:2.0:1767319485.383830:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826624 00000100:00100000:2.0:1767319485.383839:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982826880 00000100:00100000:2.0:1767319485.383840:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982826880 00000100:00100000:2.0:1767319485.383844:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827136 00000100:00100000:2.0:1767319485.383846:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827136 00000100:00100000:2.0:1767319485.383851:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1503 00000100:00100000:2.0:1767319485.383854:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982826624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319485.383875:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319485.383902:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982826624:12345-0@lo:38:kworker.0 Request processed in 46us (249us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.383907:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1503 00000100:00100000:2.0:1767319485.383918:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1504 00000100:00100000:2.0:1767319485.383921:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982826880:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.383925:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1505 00010000:00020000:2.0:1767319485.383929:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319485.383930:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982827136:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319485.383951:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982826880:12345-0@lo:38:kworker.0 Request processed in 28us (264us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.383957:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1504 00010000:00020000:1.0:1767319485.383960:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319485.383982:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319485.383986:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319485.383987:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982827136:12345-0@lo:38:kworker.0 Request processed in 56us (254us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319485.383990:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982826624:0@lo:38:kworker.0 00000100:00100000:1.0:1767319485.383993:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1505 00000100:00080000:2.0:1767319485.383997:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319485.383999:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319485.384002:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982826880:0@lo:38:kworker.0 00000100:00080000:2.0:1767319485.384008:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319485.384011:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319485.384014:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982827008:0@lo:38:kworker.0 00000100:00080000:2.0:1767319485.384020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319485.384023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319485.384026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982827136:0@lo:38:kworker.0 00000100:00100000:3.0:1767319485.384616:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982826496:12345-0@lo:400:kworker.0 Request processed in 1858us (2049us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319485.384624:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1142 00010000:00080000:0.0:1767319485.384856:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1500 x1853163982826496/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319501 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319485.384875:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982826496:0@lo:400:kworker.0 00010000:00010000:1.0:1767319485.572989:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319486.348672:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319486.348744:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627776 00000100:00100000:2.0:1767319486.348754:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627776 00000100:00100000:2.0:1767319486.348906:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1144 00000100:00100000:2.0:1767319486.348917:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915627776:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319486.349000:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915627776:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (338us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319486.349008:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1144 00000100:00080000:1.0:1767319486.349222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:2.0:1767319486.353400:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915627520 offset 0 length 520: 4 00000100:00100000:2.0:1767319486.353463:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319486.353769:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627648 00000100:00100000:0.0:1767319486.353779:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627648 00000100:00100000:0.0:1767319486.353830:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1506 00000100:00100000:0.0:1767319486.353838:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915627648:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319486.353875:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319486.353940:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915627648:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 101us (475us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319486.353947:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1506 00010000:00010000:1.0:1767319486.598715:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319487.620536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319488.644567:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319489.476712:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319464/real 1767319464] req@ffff934633015880 x1853163982823040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319489 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319489.476759:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934633015880 x1853163982823040/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319489 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319489.476775:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319489.476839:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319489.476844:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982823040:0@lo:250:kworker.0 00000100:00080000:1.0:1767319489.476944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319489.476951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319489.476955:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319489.476960:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4713 00000100:00080000:1.0:1767319489.476964:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319489.476972:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319489.477023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163982827264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319489.477081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319489.477085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319489.477087:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319489.477090:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4733 00000100:00080000:1.0:1767319489.477092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319489.477095:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319489.477109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982827520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319489.477118:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767319489.477135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319489.477137:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982827392:0@lo:400:kworker.0 00000100:00080000:1.0:1767319489.477140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319489.477145:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319489.477149:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4733 00000100:00080000:1.0:1767319489.477151:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319489.477154:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319489.477166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014700 x1853163982827776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319489.477174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319489.477178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319489.477180:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319489.477183:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4733 00000100:00080000:1.0:1767319489.477185:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319489.477189:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319489.477217:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319489.477309:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319489.477318:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982827648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319489.477344:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319489.477373:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827392 00000100:00100000:0.0:1767319489.477380:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827392 00000100:00080000:3.0:1767319489.477417:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319489.477441:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827648 00000100:00100000:0.0:1767319489.477444:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827648 00000100:00100000:0.0:1767319489.477491:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145 00000100:00100000:0.0:1767319489.477500:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982827392:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319489.477530:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319489.477555:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982827392:12345-0@lo:400:kworker.0 Request processed in 54us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319489.477562:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1146 00000100:00100000:2.0:1767319489.477613:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982827264:0@lo:250:kworker.0 00000400:00000100:2.0:1767319489.477638:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982827264 offset 0 length 520: 4 00000100:00100000:2.0:1767319489.477658:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982827520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319489.477681:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319489.477698:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982827776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319489.477716:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319489.477740:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827520 00000100:00100000:2.0:1767319489.477744:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827520 00000100:00100000:3.0:1767319489.477773:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982827648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319489.477829:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982827648:12345-0@lo:400:kworker.0 Request processed in 250us (475us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319489.477873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163982827904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319489.477908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319489.477911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319489.477913:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319489.477917:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4733 00000100:00080000:1.0:1767319489.477919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319489.477923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319489.477932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911500 x1853163982828288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319489.477949:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319489.477956:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982828032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.477987:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319489.478032:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828032 00000100:00100000:1.0:1767319489.478036:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828032 00000100:00100000:1.0:1767319489.478066:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 715 00000100:00100000:1.0:1767319489.478072:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982828032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.478103:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982828032:12345-0@lo:400:kworker.0 Request processed in 32us (116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319489.478109:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 715 00000100:00100000:1.0:1767319489.478146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982828032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.478159:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319489.478165:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982828160:0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.478185:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319489.479116:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828160 00000100:00100000:1.0:1767319489.479124:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828160 00000100:00100000:1.0:1767319489.479163:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00100000:1.0:1767319489.479169:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982828160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.479231:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982828160:12345-0@lo:400:kworker.0 Request processed in 62us (1044us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319489.479239:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00000100:00080000:1.0:1767319489.479259:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319489.479263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319489.479301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982828160:0@lo:400:kworker.0 00010000:00080000:1.0:1767319489.479346:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8e00 x1853163982827392/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319505 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319489.479355:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982827392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319489.479382:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827776 00000100:00100000:1.0:1767319489.479385:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827776 00000100:00100000:0.0:1767319489.484924:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145 00000100:00100000:0.0:1767319489.485024:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982827904:0@lo:38:kworker.0 00000100:00100000:0.0:1767319489.485102:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319489.485141:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982828288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319489.485166:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319489.485199:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828288 00000100:00100000:0.0:1767319489.485207:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828288 00000100:00100000:0.0:1767319489.485231:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1507 00000100:00100000:0.0:1767319489.485238:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982828288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319489.485290:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319489.485334:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982828288:12345-0@lo:38:kworker.0 Request processed in 95us (166us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319489.485342:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1507 00000100:00080000:0.0:1767319489.485373:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319489.485378:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319489.485381:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982828288:0@lo:38:kworker.0 00010000:00080000:0.0:1767319489.485444:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163982827648/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319505 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319489.485457:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982827648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319489.485501:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982827904 00000100:00100000:0.0:1767319489.485505:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982827904 00000100:00100000:0.0:1767319489.485526:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 402 00000100:00100000:0.0:1767319489.485530:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982827904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319489.485541:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319489.485571:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982827904:12345-0@lo:38:kworker.0 Request processed in 39us (470us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319489.485578:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 402 00000100:00080000:0.0:1767319489.485598:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319489.485603:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319489.485606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982827904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319489.496831:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1508 00000100:00100000:2.0:1767319489.496845:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982827520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319489.496871:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319489.496960:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982827520:12345-0@lo:38:kworker.0 Request processed in 117us (19278us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319489.496967:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1508 00000100:00080000:0.0:1767319489.497183:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319489.497190:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319489.497194:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982827520:0@lo:38:kworker.0 00000100:00100000:3.0:1767319489.497977:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1146 00000100:00100000:3.0:1767319489.500191:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1509 00000100:00100000:3.0:1767319489.500301:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982827776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319489.500325:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319489.500415:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982827776:12345-0@lo:38:kworker.0 Request processed in 115us (22697us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319489.500422:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1509 00000100:00080000:0.0:1767319489.500700:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319489.500708:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319489.500712:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982827776:0@lo:38:kworker.0 00010000:00010000:1.0:1767319489.671859:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319489.988542:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982828416:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319489.988616:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319489.988758:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828416 00000100:00100000:3.0:1767319489.988767:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828416 00000100:00100000:3.0:1767319489.988831:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 778 00000100:00100000:3.0:1767319489.988852:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982828416:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319489.988882:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319489.988943:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982828416:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (327us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319489.988951:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 778 00000100:00100000:0.0:1767319489.989566:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319489.989578:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982828544:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319489.989613:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319489.989739:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828544 00000100:00100000:2.0:1767319489.989745:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828544 00000100:00100000:2.0:1767319489.989793:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 779 00000100:00100000:2.0:1767319489.989801:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982828544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319489.989821:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319489.989859:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982828544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (244us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319489.989866:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 779 00000100:00100000:0.0:1767319489.989952:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982828544:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319489.990052:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982828416:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319490.692449:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319491.468246:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319491.468358:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628032 00000100:00100000:1.0:1767319491.468369:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628032 00000100:00100000:1.0:1767319491.468505:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1147 00000100:00100000:1.0:1767319491.468516:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915628032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319491.468601:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915628032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319491.468609:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1147 00000100:00080000:3.0:1767319491.473973:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319491.479495:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319491.479758:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915627904 00000100:00100000:2.0:1767319491.479769:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915627904 00000100:00100000:2.0:1767319491.479823:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1510 00000100:00100000:2.0:1767319491.479831:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915627904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319491.479854:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319491.479924:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915627904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319491.479933:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1510 00010000:00010000:3.0:1767319491.716506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319492.740911:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319493.764592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319494.596631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319494.596640:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319494.596718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319494.596723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319494.596726:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319494.596734:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4738 00000100:00080000:0.0:1767319494.596738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319494.596744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319494.596766:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92300 x1853163982828800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319494.596805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319494.596809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319494.596811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319494.596815:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4738 00000100:00080000:0.0:1767319494.596818:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319494.596821:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319494.596831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93800 x1853163982829056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319494.596839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319494.596841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319494.596842:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319494.596846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4738 00000100:00080000:0.0:1767319494.596847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319494.596851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319494.596870:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91880 x1853163982829184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319494.596892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319494.596895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319494.596896:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319494.596899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4738 00000100:00080000:0.0:1767319494.596901:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319494.596905:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319494.596914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163982829568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319494.596937:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982828928:0@lo:400:kworker.0 00000100:00100000:0.0:1767319494.596999:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319494.597033:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319494.597039:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982829312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319494.597062:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319494.597208:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982829440:0@lo:400:kworker.0 00000100:00100000:3.0:1767319494.597236:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319494.597264:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319494.597291:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982828672:0@lo:400:kworker.0 00000100:00100000:3.0:1767319494.597325:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319494.597545:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828928 00000100:00100000:1.0:1767319494.597554:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828928 00000100:00100000:1.0:1767319494.597608:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828672 00000100:00100000:1.0:1767319494.597611:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828672 00000100:00100000:1.0:1767319494.597633:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148 00000100:00100000:1.0:1767319494.597642:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982828928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.597703:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982828928:12345-0@lo:400:kworker.0 Request processed in 62us (703us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319494.597710:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148 00000100:00100000:1.0:1767319494.597728:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1149 00000100:00100000:1.0:1767319494.597735:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982828672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.597767:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982828672:12345-0@lo:400:kworker.0 Request processed in 33us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319494.597773:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1149 00000100:00100000:1.0:1767319494.597791:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829312 00000100:00100000:1.0:1767319494.597795:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829312 00000100:00100000:1.0:1767319494.597817:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829440 00000100:00100000:1.0:1767319494.597821:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829440 00000100:00100000:1.0:1767319494.597840:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 717 00000100:00100000:1.0:1767319494.597845:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982829312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.597952:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982829312:12345-0@lo:400:kworker.0 Request processed in 106us (889us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319494.597959:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 717 00000100:00100000:1.0:1767319494.597970:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 718 00000100:00100000:1.0:1767319494.597977:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982829440:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.598002:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982829440:12345-0@lo:400:kworker.0 Request processed in 26us (765us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319494.598008:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 718 00010000:00080000:1.0:1767319494.598047:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d92a00 x1853163982828672/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319510 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319494.598061:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982828672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.598072:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982829440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.598101:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982829312:0@lo:400:kworker.0 00010000:00080000:1.0:1767319494.598111:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d90e00 x1853163982828928/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319510 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319494.598120:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982828928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319494.598138:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982828800:0@lo:38:kworker.0 00000100:00100000:1.0:1767319494.598165:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319494.598182:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982829056:0@lo:38:kworker.0 00000100:00100000:0.0:1767319494.598407:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982828800 00000100:00100000:0.0:1767319494.598411:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982828800 00000100:00100000:0.0:1767319494.598432:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1511 00000100:00100000:0.0:1767319494.598437:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982828800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319494.598454:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319494.598485:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982828800:12345-0@lo:38:kworker.0 Request processed in 46us (318us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319494.598491:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1511 00000100:00100000:1.0:1767319494.598656:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319494.598721:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982829184:0@lo:38:kworker.0 00000100:00100000:1.0:1767319494.598746:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319494.598768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982829568:0@lo:38:kworker.0 00000100:00100000:1.0:1767319494.598786:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319494.598823:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829184 00000100:00100000:0.0:1767319494.598827:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829184 00000100:00100000:0.0:1767319494.598848:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 403 00000100:00100000:0.0:1767319494.598853:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982829184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319494.598874:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319494.598901:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982829184:12345-0@lo:38:kworker.0 Request processed in 47us (154us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319494.598907:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 403 00000100:00080000:3.0:1767319494.599205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319494.599210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319494.599213:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319494.600907:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319494.600916:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319494.600920:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982828800:0@lo:38:kworker.0 00000100:00080000:1.0:1767319494.600932:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319494.600936:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319494.600939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982829184:0@lo:38:kworker.0 00000100:00100000:2.0:1767319494.601324:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829056 00000100:00100000:2.0:1767319494.601332:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829056 00000100:00100000:3.0:1767319494.601427:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829568 00000100:00100000:3.0:1767319494.601431:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829568 00000100:00100000:3.0:1767319494.601481:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1512 00000100:00100000:3.0:1767319494.601489:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982829056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319494.601507:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319494.601566:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982829056:12345-0@lo:38:kworker.0 Request processed in 76us (2908us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319494.601574:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1512 00000100:00100000:3.0:1767319494.601588:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1513 00000100:00100000:3.0:1767319494.601592:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982829568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319494.601600:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319494.601623:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982829568:12345-0@lo:38:kworker.0 Request processed in 30us (2836us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319494.601629:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1513 00000100:00080000:1.0:1767319494.601817:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319494.601822:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319494.601826:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982829056:0@lo:38:kworker.0 00000100:00080000:1.0:1767319494.601834:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319494.601838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319494.601841:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982829568:0@lo:38:kworker.0 00010000:00010000:3.0:1767319494.789422:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319495.108945:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319495.108968:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982829696:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319495.109049:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319495.109164:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829696 00000100:00100000:3.0:1767319495.109182:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829696 00000100:00100000:3.0:1767319495.109235:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 780 00000100:00100000:3.0:1767319495.109245:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982829696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319495.109307:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319495.109357:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982829696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 114us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319495.109365:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 780 00000100:00100000:1.0:1767319495.109699:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982829696:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319495.109946:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319495.110627:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982829824:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319495.110676:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319495.111263:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829824 00000100:00100000:2.0:1767319495.111284:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829824 00000100:00100000:2.0:1767319495.111358:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 781 00000100:00100000:2.0:1767319495.111365:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982829824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319495.111395:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319495.111457:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982829824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (772us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319495.111466:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 781 00000100:00100000:1.0:1767319495.111542:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982829824:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319495.812487:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319496.590675:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319496.590746:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319496.590950:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628288 00000100:00100000:1.0:1767319496.590961:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628288 00000100:00100000:1.0:1767319496.591090:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628160 00000100:00100000:1.0:1767319496.591095:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628160 00000100:00100000:1.0:1767319496.591117:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1150 00000100:00100000:1.0:1767319496.591128:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915628288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319496.591210:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915628288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (556us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319496.591219:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1150 00000100:00100000:1.0:1767319496.591242:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1514 00000100:00100000:1.0:1767319496.591247:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915628160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319496.591314:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319496.591339:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915628160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (593us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319496.591345:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1514 00000100:00080000:3.0:1767319496.592507:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319496.838402:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319497.860550:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319498.884931:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319499.716453:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319499.716464:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319499.716526:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319499.716531:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319499.716535:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319499.716542:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4743 00000100:00080000:3.0:1767319499.716546:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319499.716553:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319499.716575:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6d80 x1853163982830080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319499.716649:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319499.716653:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319499.716655:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319499.716659:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4743 00000100:00080000:3.0:1767319499.716661:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319499.716666:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319499.716676:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321c00 x1853163982830336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319499.716685:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319499.716689:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319499.716690:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319499.716693:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4743 00000100:00080000:3.0:1767319499.716695:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319499.716699:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319499.716707:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322680 x1853163982830464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319499.716726:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319499.716730:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319499.716732:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319499.716735:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4743 00000100:00080000:3.0:1767319499.716737:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319499.716741:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319499.716749:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323b80 x1853163982830848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319499.716855:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982830080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319499.716916:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319499.716964:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982830336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319499.716984:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319499.716997:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982830464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319499.717019:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319499.717049:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982830848:0@lo:38:kworker.0 00000100:00100000:2.0:1767319499.717066:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319499.717378:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830464 00000100:00100000:0.0:1767319499.717386:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830464 00000100:00100000:0.0:1767319499.717429:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 404 00000100:00100000:0.0:1767319499.717436:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982830464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319499.717456:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319499.717502:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982830464:12345-0@lo:38:kworker.0 Request processed in 64us (480us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319499.717508:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 404 00000100:00100000:0.0:1767319499.717531:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982830720:0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.717557:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319499.717578:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319499.717583:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982829952:0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.717615:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319499.717640:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830080 00000100:00100000:0.0:1767319499.717644:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830080 00000100:00100000:0.0:1767319499.717661:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830336 00000100:00100000:0.0:1767319499.717664:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830336 00000100:00100000:0.0:1767319499.717671:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830848 00000100:00100000:0.0:1767319499.717673:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830848 00000100:00100000:0.0:1767319499.717680:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982829952 00000100:00100000:0.0:1767319499.717682:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982829952 00000100:00100000:0.0:1767319499.717711:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151 00000100:00100000:0.0:1767319499.717717:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982829952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.717756:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982829952:12345-0@lo:400:kworker.0 Request processed in 39us (139us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319499.717762:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151 00000100:00100000:0.0:1767319499.717774:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1515 00000100:00100000:0.0:1767319499.717778:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319499.717786:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319499.717807:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830080:12345-0@lo:38:kworker.0 Request processed in 28us (891us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319499.717813:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1515 00000100:00100000:0.0:1767319499.717822:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1516 00000100:00100000:0.0:1767319499.717826:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319499.717834:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319499.717856:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830336:12345-0@lo:38:kworker.0 Request processed in 29us (871us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319499.717861:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1516 00000100:00100000:0.0:1767319499.717870:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1517 00000100:00100000:0.0:1767319499.717874:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319499.717882:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319499.717903:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982830848:12345-0@lo:38:kworker.0 Request processed in 27us (835us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319499.717908:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1517 00000100:00080000:0.0:1767319499.717923:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319499.717933:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830720 00000100:00100000:0.0:1767319499.717936:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830720 00000100:00100000:0.0:1767319499.717962:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00100000:0.0:1767319499.717968:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982830720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.717997:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982830720:12345-0@lo:400:kworker.0 Request processed in 29us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319499.718003:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00080000:0.0:1767319499.718017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:0.0:1767319499.718040:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6680 x1853163982829952/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319515 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319499.718051:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982829952:0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.718060:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982830720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319499.718088:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982830592:0@lo:400:kworker.0 00000100:00080000:0.0:1767319499.718092:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319499.718096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319499.718099:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982830080:0@lo:38:kworker.0 00000100:00080000:0.0:1767319499.718106:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319499.718109:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319499.718112:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982830336:0@lo:38:kworker.0 00000100:00100000:1.0:1767319499.718117:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319499.718119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319499.718122:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319499.718125:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982830464:0@lo:38:kworker.0 00000100:00080000:0.0:1767319499.718132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319499.718134:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319499.718137:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982830848:0@lo:38:kworker.0 00000100:00100000:1.0:1767319499.718138:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319499.718143:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982830208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319499.718161:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319499.718185:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830208 00000100:00100000:1.0:1767319499.718185:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830592 00000100:00100000:0.0:1767319499.718188:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830208 00000100:00100000:1.0:1767319499.718188:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830592 00000100:00100000:0.0:1767319499.718213:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152 00000100:00100000:1.0:1767319499.718213:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 00000100:00100000:0.0:1767319499.718219:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982830208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319499.718219:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982830592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.718652:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982830208:12345-0@lo:400:kworker.0 Request processed in 32us (89us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319499.718659:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152 00000100:00080000:0.0:1767319499.718675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319499.718678:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319499.727918:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982830592:12345-0@lo:400:kworker.0 Request processed in 9687us (9788us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319499.727936:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00010000:00080000:0.0:1767319499.730344:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323800 x1853163982830208/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319515 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319499.730363:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982830208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319499.730394:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982830592:0@lo:400:kworker.0 00010000:00010000:1.0:1767319499.910938:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319500.228717:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319500.228744:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982830976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319500.228810:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319500.229100:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982830976 00000100:00100000:3.0:1767319500.229110:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982830976 00000100:00100000:3.0:1767319500.229165:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 782 00000100:00100000:3.0:1767319500.229175:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982830976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319500.229204:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319500.229249:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982830976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319500.229256:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 782 00000100:00100000:0.0:1767319500.229409:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982830976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319500.230245:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982831104:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319500.230329:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319500.230434:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831104 00000100:00100000:3.0:1767319500.230441:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831104 00000100:00100000:3.0:1767319500.230482:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 783 00000100:00100000:3.0:1767319500.230490:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982831104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319500.230511:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319500.230979:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982831104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 489us (648us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319500.230987:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 783 00000100:00100000:0.0:1767319500.231377:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982831104:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767319500.932495:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319501.711554:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319501.711650:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319501.711749:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628544 00000100:00100000:0.0:1767319501.711759:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628544 00000100:00100000:0.0:1767319501.711833:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628416 00000100:00100000:0.0:1767319501.711837:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628416 00000100:00100000:0.0:1767319501.711858:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1153 00000100:00100000:0.0:1767319501.711867:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915628544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319501.711932:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915628544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 67us (406us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319501.711940:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1153 00000100:00100000:0.0:1767319501.711955:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1518 00000100:00100000:0.0:1767319501.711959:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915628416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319501.711979:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319501.711998:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915628416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 38us (348us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319501.712004:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1518 00000100:00080000:1.0:1767319501.712147:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319501.956683:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319502.980479:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319504.006816:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319504.836823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319504.836833:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319504.836892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319504.836899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319504.836903:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319504.836910:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4748 00000100:00080000:0.0:1767319504.836916:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319504.836923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319504.836943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90e00 x1853163982831360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319504.836984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319504.836988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319504.836990:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319504.836994:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4748 00000100:00080000:0.0:1767319504.837004:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319504.837009:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319504.837020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90000 x1853163982831616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319504.837029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319504.837032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319504.837034:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319504.837037:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4748 00000100:00080000:0.0:1767319504.837039:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319504.837043:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319504.837053:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163982831744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319504.837088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319504.837092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319504.837094:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319504.837097:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4748 00000100:00080000:0.0:1767319504.837100:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319504.837103:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319504.837136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1f80 x1853163982832128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319504.837168:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982831488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319504.837228:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319504.837311:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319504.837318:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982831872:0@lo:400:kworker.0 00000100:00100000:0.0:1767319504.837343:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319504.837381:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831488 00000100:00100000:0.0:1767319504.837387:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831488 00000100:00100000:0.0:1767319504.837436:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1154 00000100:00100000:0.0:1767319504.837931:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982831488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319504.838078:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982831488:12345-0@lo:400:kworker.0 Request processed in 634us (848us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319504.838086:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1154 00000100:00100000:0.0:1767319504.838113:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831872 00000100:00100000:0.0:1767319504.838117:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831872 00000100:00100000:0.0:1767319504.838147:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 721 00000100:00100000:0.0:1767319504.838154:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982831872:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319504.838182:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982831872:12345-0@lo:400:kworker.0 Request processed in 29us (839us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319504.838188:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 721 00000100:00080000:0.0:1767319504.838202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319504.838207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319504.838235:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982831872:0@lo:400:kworker.0 00010000:00080000:0.0:1767319504.838246:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d92d80 x1853163982831488/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319520 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319504.838256:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982831488:0@lo:400:kworker.0 00000100:00100000:3.0:1767319504.838488:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982832000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319504.838528:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319504.838562:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319504.838568:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982831232:0@lo:400:kworker.0 00000100:00100000:3.0:1767319504.838589:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319504.838621:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831232 00000100:00100000:3.0:1767319504.838625:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831232 00000100:00100000:3.0:1767319504.838874:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1155 00000100:00100000:3.0:1767319504.838883:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982831232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319504.838917:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982831232:12345-0@lo:400:kworker.0 Request processed in 37us (327us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319504.838923:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1155 00000100:00080000:3.0:1767319504.838943:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319504.839011:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d92300 x1853163982831232/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319520 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319504.839020:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982831232:0@lo:400:kworker.0 00000100:00100000:2.0:1767319504.839218:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982831360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319504.839247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319504.839291:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982831616:0@lo:38:kworker.0 00000100:00100000:2.0:1767319504.839311:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319504.839331:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982831744:0@lo:38:kworker.0 00000100:00100000:2.0:1767319504.839352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319504.839373:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982832128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319504.839392:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319504.839418:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831744 00000100:00100000:2.0:1767319504.839422:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831744 00000100:00100000:2.0:1767319504.839458:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 405 00000100:00100000:2.0:1767319504.839463:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982831744:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319504.839480:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319504.839522:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982831744:12345-0@lo:38:kworker.0 Request processed in 58us (168us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319504.839527:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 405 00000100:00100000:2.0:1767319504.839547:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831360 00000100:00100000:2.0:1767319504.839551:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831360 00000100:00100000:2.0:1767319504.839568:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982831616 00000100:00100000:2.0:1767319504.839570:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982831616 00000100:00100000:2.0:1767319504.839582:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832128 00000100:00100000:2.0:1767319504.839585:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832128 00000100:00100000:2.0:1767319504.839594:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1519 00000100:00100000:2.0:1767319504.839599:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982831360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319504.839608:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319504.839632:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982831360:12345-0@lo:38:kworker.0 Request processed in 33us (385us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319504.839638:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1519 00000100:00100000:2.0:1767319504.839648:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1520 00000100:00100000:2.0:1767319504.839652:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982831616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319504.839660:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319504.839892:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982831616:12345-0@lo:38:kworker.0 Request processed in 237us (579us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319504.839898:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1520 00000100:00100000:2.0:1767319504.839909:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1521 00000100:00100000:2.0:1767319504.839913:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982832128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319504.839921:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319504.839942:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982832128:12345-0@lo:38:kworker.0 Request processed in 28us (548us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319504.839948:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1521 00000100:00080000:2.0:1767319504.839990:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319504.839995:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319504.840145:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982831360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319504.840157:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319504.840160:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319504.840163:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982831616:0@lo:38:kworker.0 00000100:00080000:2.0:1767319504.840170:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319504.840172:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319504.840175:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982831744:0@lo:38:kworker.0 00000100:00080000:2.0:1767319504.840182:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319504.840185:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319504.840187:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982832128:0@lo:38:kworker.0 00000100:00100000:0.0:1767319504.840268:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832000 00000100:00100000:0.0:1767319504.840291:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832000 00000100:00100000:0.0:1767319504.840333:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767319504.840340:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982832000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319504.840376:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982832000:12345-0@lo:400:kworker.0 Request processed in 37us (1847us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319504.840383:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000100:00100000:3.0:1767319504.840922:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982832000:0@lo:400:kworker.0 00000100:00080000:2.0:1767319504.842942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319505.035058:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319505.348625:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319505.348647:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982832256:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319505.348725:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319505.348899:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832256 00000100:00100000:3.0:1767319505.348908:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832256 00000100:00100000:3.0:1767319505.348961:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 784 00000100:00100000:3.0:1767319505.348971:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982832256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319505.349010:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319505.349059:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982832256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319505.349067:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 784 00000100:00100000:0.0:1767319505.349422:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982832256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319505.349638:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319505.349649:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982832384:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319505.349693:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319505.349822:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832384 00000100:00100000:2.0:1767319505.349826:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832384 00000100:00100000:2.0:1767319505.349860:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 785 00000100:00100000:2.0:1767319505.349867:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982832384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319505.349885:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319505.349926:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982832384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319505.349932:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 785 00000100:00100000:1.0:1767319505.350112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982832384:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319506.052550:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319506.831431:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319506.831484:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319506.831659:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628800 00000100:00100000:2.0:1767319506.831669:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628800 00000100:00100000:3.0:1767319506.831681:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915628672 00000100:00100000:3.0:1767319506.831685:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915628672 00000100:00100000:3.0:1767319506.831726:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1156 00000100:00100000:2.0:1767319506.831733:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1522 00000100:00100000:3.0:1767319506.831736:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915628800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319506.831738:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915628672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319506.831757:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319506.831805:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915628672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 65us (319us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319506.831812:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1522 00000100:00080000:0.0:1767319506.831954:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319506.832833:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915628800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1093us (1408us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319506.832843:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1156 00010000:00010000:3.0:1767319507.076645:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319508.100454:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319509.126240:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319509.956483:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319509.956506:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319509.956572:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319509.956578:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767319509.956581:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319509.956583:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319509.956594:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4753 00000100:00080000:3.0:1767319509.956598:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319509.956605:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982832512:0@lo:400:kworker.0 00000100:00080000:3.0:1767319509.956608:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319509.956636:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322a00 x1853163982832640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319509.956685:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319509.956842:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832512 00000100:00100000:2.0:1767319509.956851:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832512 00000100:00080000:3.0:1767319509.956898:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319509.956902:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319509.956904:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319509.956909:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4753 00000100:00080000:3.0:1767319509.956912:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319509.956917:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319509.956925:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157 00000100:00100000:3.0:1767319509.956932:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982832896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319509.956940:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982832512:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319509.956947:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319509.956951:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319509.956953:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319509.956957:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4753 00000100:00080000:3.0:1767319509.956960:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319509.956963:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319509.956977:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320380 x1853163982833024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319509.957013:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319509.957017:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982832512:12345-0@lo:400:kworker.0 Request processed in 82us (331us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319509.957018:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319509.957022:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319509.957029:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4753 00000100:00100000:2.0:1767319509.957032:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157 00000100:00080000:3.0:1767319509.957034:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319509.957049:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319509.957065:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982833408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319509.957068:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319509.957081:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982833280:0@lo:400:kworker.0 00000100:00100000:2.0:1767319509.957127:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319509.957142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319509.957160:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982832768:0@lo:400:kworker.0 00000100:00100000:0.0:1767319509.957193:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319509.957222:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319509.957228:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982833152:0@lo:400:kworker.0 00000100:00100000:0.0:1767319509.957250:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319509.957291:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832768 00000100:00100000:2.0:1767319509.957296:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832768 00000100:00100000:0.0:1767319509.957302:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833280 00000100:00100000:0.0:1767319509.957308:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833280 00000100:00100000:2.0:1767319509.957338:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1158 00000100:00100000:0.0:1767319509.957345:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833152 00000100:00100000:2.0:1767319509.957347:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982832768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319509.957349:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833152 00000100:00100000:0.0:1767319509.957376:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00100000:0.0:1767319509.957384:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982833280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319509.957437:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982833280:12345-0@lo:400:kworker.0 Request processed in 54us (310us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319509.957444:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00000100:00100000:0.0:1767319509.957456:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 724 00000100:00100000:0.0:1767319509.957461:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982833152:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319509.957541:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982832640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319509.957573:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982833280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319509.960202:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982833152:12345-0@lo:400:kworker.0 Request processed in 2729us (2940us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319509.960217:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 724 00000100:00080000:0.0:1767319509.960262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319509.960268:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319509.960286:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319509.962761:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982832768:12345-0@lo:400:kworker.0 Request processed in 5405us (5555us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319509.962777:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1158 00000100:00100000:1.0:1767319509.964080:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319509.964161:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982833152:0@lo:400:kworker.0 00010000:00080000:0.0:1767319509.964181:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323100 x1853163982832768/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319525 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319509.964192:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982832768:0@lo:400:kworker.0 00000100:00100000:2.0:1767319509.964665:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832640 00000100:00100000:2.0:1767319509.964673:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832640 00000100:00100000:2.0:1767319509.964725:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1523 00000100:00100000:2.0:1767319509.964734:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982832640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319509.964755:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319509.964802:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982832640:12345-0@lo:38:kworker.0 Request processed in 68us (726us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319509.964808:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1523 00000100:00080000:1.0:1767319509.966740:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319509.966752:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319509.966757:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982832640:0@lo:38:kworker.0 00000100:00100000:1.0:1767319509.966769:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982832896:0@lo:38:kworker.0 00000100:00100000:1.0:1767319509.966824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319509.966856:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982833024:0@lo:38:kworker.0 00000100:00100000:1.0:1767319509.966881:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319509.966915:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982833408:0@lo:38:kworker.0 00000100:00100000:1.0:1767319509.966936:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319509.967158:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833024 00000100:00100000:0.0:1767319509.967166:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833024 00000100:00100000:0.0:1767319509.967215:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 406 00000100:00100000:0.0:1767319509.967222:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982833024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319509.967242:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319509.967315:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982833024:12345-0@lo:38:kworker.0 Request processed in 91us (433us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319509.967326:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 406 00000100:00100000:0.0:1767319509.967350:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982832896 00000100:00100000:0.0:1767319509.967354:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982832896 00000100:00100000:0.0:1767319509.967375:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833408 00000100:00100000:0.0:1767319509.967378:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833408 00000100:00100000:0.0:1767319509.967393:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1524 00000100:00100000:0.0:1767319509.967398:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982832896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319509.967418:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319509.967449:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982832896:12345-0@lo:38:kworker.0 Request processed in 49us (624us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319509.967455:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1524 00000100:00100000:0.0:1767319509.967466:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1525 00000100:00100000:0.0:1767319509.967470:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982833408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319509.967479:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319509.967501:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982833408:12345-0@lo:38:kworker.0 Request processed in 30us (564us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319509.967507:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1525 00000100:00080000:1.0:1767319509.967599:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319509.967604:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319509.967608:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982832896:0@lo:38:kworker.0 00000100:00080000:1.0:1767319509.967616:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319509.967619:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319509.967621:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982833024:0@lo:38:kworker.0 00000100:00080000:1.0:1767319509.967629:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319509.967631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319509.967633:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982833408:0@lo:38:kworker.0 00010000:00080000:3.0:1767319509.971323:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320700 x1853163982832512/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319525 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319509.971345:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982832512:0@lo:400:kworker.0 00010000:00010000:3.0:1767319510.148543:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319510.468574:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982833536:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319510.468654:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319510.468748:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833536 00000100:00100000:3.0:1767319510.468760:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833536 00000100:00100000:3.0:1767319510.468812:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 786 00000100:00100000:3.0:1767319510.468821:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982833536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319510.468852:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319510.468864:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982833664:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319510.468901:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319510.468958:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833664 00000100:00100000:0.0:1767319510.468964:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833664 00000100:00100000:0.0:1767319510.468996:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 787 00000100:00100000:0.0:1767319510.469004:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982833664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319510.469021:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319510.469064:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982833664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319510.469071:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 787 00000100:00100000:2.0:1767319510.469223:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982833664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319510.469374:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982833536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 553us (719us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319510.469382:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 786 00000100:00100000:1.0:1767319510.469590:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982833536:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319510.926556:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319510.926586:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915628928 offset 0 length 520: 4 00000100:00100000:3.0:1767319510.926606:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319510.926682:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629184 00000100:00100000:0.0:1767319510.926692:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629184 00000100:00100000:0.0:1767319510.926765:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629056 00000100:00100000:0.0:1767319510.926768:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629056 00000100:00100000:0.0:1767319510.926789:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1159 00000100:00100000:0.0:1767319510.926799:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915629184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319510.926873:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915629184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319510.926881:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1159 00000100:00100000:0.0:1767319510.926899:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1526 00000100:00100000:0.0:1767319510.926904:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915629056:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319510.926924:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319510.926948:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915629056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (342us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319510.926955:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1526 00000100:00080000:1.0:1767319510.927025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319511.172666:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319512.196665:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319513.221913:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319514.054475:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319489/real 1767319489] req@ffff9346076fb100 x1853163982827264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319514 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319514.054508:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076fb100 x1853163982827264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319514 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319514.054523:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319514.054552:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319514.054556:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982827264:0@lo:250:kworker.0 00000100:00080000:0.0:1767319514.055224:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319514.055232:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319514.055234:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319514.055239:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4738 00000100:00080000:0.0:1767319514.055242:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319514.055249:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319514.055306:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0e00 x1853163982833792/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319514.055383:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319514.055386:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319514.055388:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319514.055393:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4758 00000100:00080000:0.0:1767319514.055395:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319514.055399:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319514.055413:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3b80 x1853163982834048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319514.055436:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319514.055440:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319514.055441:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319514.055445:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4758 00000100:00080000:0.0:1767319514.055446:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319514.055450:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319514.055460:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2d80 x1853163982834304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319514.055468:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319514.055471:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319514.055472:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319514.055475:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4758 00000100:00080000:0.0:1767319514.055477:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319514.055481:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319514.055490:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1880 x1853163982834432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319514.055507:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319514.055510:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319514.055512:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319514.055515:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4758 00000100:00080000:0.0:1767319514.055518:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319514.055521:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319514.055530:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1500 x1853163982834816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319514.055554:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982834176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319514.055614:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319514.055685:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319514.055692:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982834560:0@lo:400:kworker.0 00000100:00100000:0.0:1767319514.055728:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319514.055758:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834560 00000100:00100000:0.0:1767319514.055766:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834560 00000100:00100000:0.0:1767319514.055814:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 725 00000100:00100000:0.0:1767319514.055823:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982834560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319514.055868:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982834560:12345-0@lo:400:kworker.0 Request processed in 46us (139us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319514.055875:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 725 00000100:00100000:0.0:1767319514.055911:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982834560:0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056209:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982834688:0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056240:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319514.056267:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319514.056299:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982833920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056321:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319514.056369:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834176 00000100:00100000:3.0:1767319514.056373:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834176 00000100:00100000:3.0:1767319514.056403:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982833920 00000100:00100000:3.0:1767319514.056407:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982833920 00000100:00100000:3.0:1767319514.056431:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160 00000100:00100000:3.0:1767319514.056437:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982834176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056473:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982834176:12345-0@lo:400:kworker.0 Request processed in 36us (859us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319514.056479:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160 00000100:00100000:3.0:1767319514.056494:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1161 00000100:00100000:3.0:1767319514.056499:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982833920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056528:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982833920:12345-0@lo:400:kworker.0 Request processed in 29us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319514.056535:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1161 00000100:00100000:3.0:1767319514.056553:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834688 00000100:00100000:3.0:1767319514.056557:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834688 00000100:00100000:3.0:1767319514.056581:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00100000:3.0:1767319514.056588:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982834688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056617:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982834688:12345-0@lo:400:kworker.0 Request processed in 29us (376us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319514.056623:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00010000:00080000:3.0:1767319514.056676:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0a80 x1853163982833920/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319530 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319514.056687:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982833920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319514.056696:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982834688:0@lo:400:kworker.0 00010000:00080000:3.0:1767319514.056732:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1180 x1853163982834176/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319530 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319514.056741:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982834176:0@lo:400:kworker.0 00000100:00080000:1.0:1767319514.056989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319514.056994:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319514.056996:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319514.057007:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982833792:0@lo:250:kworker.0 00000400:00000100:1.0:1767319514.057033:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982833792 offset 0 length 520: 4 00000100:00100000:1.0:1767319514.057055:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982834048:0@lo:38:kworker.0 00000100:00100000:1.0:1767319514.057091:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319514.057115:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982834304:0@lo:38:kworker.0 00000100:00100000:1.0:1767319514.057133:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319514.057156:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982834432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319514.057179:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319514.057201:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834432 00000100:00100000:1.0:1767319514.057205:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834432 00000100:00100000:1.0:1767319514.057225:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 407 00000100:00100000:1.0:1767319514.057230:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982834432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319514.057246:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319514.057293:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982834432:12345-0@lo:38:kworker.0 Request processed in 62us (113us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319514.057299:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 407 00000100:00080000:1.0:1767319514.057325:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319514.057330:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319514.057345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982834432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319514.057351:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982834816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319514.057373:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319514.057395:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834048 00000100:00100000:1.0:1767319514.057398:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834048 00000100:00100000:1.0:1767319514.057415:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834304 00000100:00100000:1.0:1767319514.057418:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834304 00000100:00100000:1.0:1767319514.057430:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834816 00000100:00100000:1.0:1767319514.057432:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834816 00000100:00100000:1.0:1767319514.057442:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1527 00000100:00100000:1.0:1767319514.057447:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319514.057458:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319514.057482:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834048:12345-0@lo:38:kworker.0 Request processed in 34us (391us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319514.057489:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1527 00000100:00100000:1.0:1767319514.057500:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1528 00000100:00100000:1.0:1767319514.057504:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319514.057512:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319514.057534:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834304:12345-0@lo:38:kworker.0 Request processed in 29us (400us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319514.057540:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1528 00000100:00100000:1.0:1767319514.057550:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1529 00000100:00100000:1.0:1767319514.057554:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319514.057562:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319514.057585:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982834816:12345-0@lo:38:kworker.0 Request processed in 30us (211us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319514.057591:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1529 00000100:00080000:1.0:1767319514.057612:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319514.057616:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319514.057618:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982834816:0@lo:38:kworker.0 00000100:00080000:1.0:1767319514.057651:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319514.057656:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319514.057658:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982834048:0@lo:38:kworker.0 00000100:00080000:1.0:1767319514.057666:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319514.057668:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319514.057671:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982834304:0@lo:38:kworker.0 00010000:00010000:1.0:1767319514.244585:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319515.269198:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319515.588690:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319515.588710:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982834944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319515.588785:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319515.588856:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982834944 00000100:00100000:3.0:1767319515.588862:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982834944 00000100:00100000:3.0:1767319515.588903:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 788 00000100:00100000:3.0:1767319515.588910:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982834944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319515.588932:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319515.588985:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982834944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (199us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319515.588993:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 788 00000100:00100000:1.0:1767319515.589113:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982834944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319515.598545:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982835072:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319515.598618:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319515.598737:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835072 00000100:00100000:2.0:1767319515.598745:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835072 00000100:00100000:2.0:1767319515.598810:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 789 00000100:00100000:2.0:1767319515.598819:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982835072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319515.598849:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319515.598900:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982835072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319515.598908:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 789 00000100:00100000:0.0:1767319515.599002:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982835072:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319516.044569:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319516.044624:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319516.044747:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629440 00000100:00100000:0.0:1767319516.044757:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629440 00000100:00100000:0.0:1767319516.044833:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629312 00000100:00100000:0.0:1767319516.044837:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629312 00000100:00100000:0.0:1767319516.044857:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1162 00000100:00100000:0.0:1767319516.044867:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915629440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319516.044873:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1530 00000100:00100000:3.0:1767319516.044879:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915629312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319516.044897:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767319516.044965:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319516.045041:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915629440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 174us (484us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319516.045049:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1162 00000100:00100000:3.0:1767319516.054420:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915629312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 9526us (9783us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319516.054434:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1530 00010000:00010000:3.0:1767319516.292559:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319517.317248:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319518.340623:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319519.172802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319519.172813:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319519.172880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319519.172884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319519.172888:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319519.172896:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4762 00000100:00080000:1.0:1767319519.172901:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319519.172908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319519.172930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982835328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319519.172979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319519.172983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319519.172985:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319519.172989:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4762 00000100:00080000:1.0:1767319519.172992:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319519.172995:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319519.173007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7100 x1853163982835584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319519.173015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319519.173017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319519.173019:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319519.173022:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4762 00000100:00080000:1.0:1767319519.173024:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319519.173027:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319519.173035:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4a80 x1853163982835712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319519.173053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319519.173057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319519.173058:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319519.173061:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4762 00000100:00080000:1.0:1767319519.173063:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319519.173066:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319519.173075:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4000 x1853163982836096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319519.173099:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982835456:0@lo:400:kworker.0 00000100:00100000:1.0:1767319519.173176:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319519.173213:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319519.173220:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982835840:0@lo:400:kworker.0 00000100:00100000:1.0:1767319519.173298:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319519.173368:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982835968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319519.173399:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319519.173431:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319519.173437:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982835200:0@lo:400:kworker.0 00000100:00100000:0.0:1767319519.173458:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319519.173486:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835840 00000100:00100000:0.0:1767319519.173494:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835840 00000100:00100000:0.0:1767319519.173549:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835968 00000100:00100000:0.0:1767319519.173553:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835968 00000100:00100000:0.0:1767319519.173575:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 727 00000100:00100000:0.0:1767319519.173582:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982835840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319519.173632:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982835840:12345-0@lo:400:kworker.0 Request processed in 50us (333us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319519.173638:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 727 00000100:00100000:0.0:1767319519.173654:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 728 00000100:00100000:0.0:1767319519.173660:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982835968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319519.173691:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982835968:12345-0@lo:400:kworker.0 Request processed in 31us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319519.173696:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 728 00000100:00080000:0.0:1767319519.173711:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319519.173715:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319519.173742:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982835968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319519.173767:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982835840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319519.173848:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835456 00000100:00100000:3.0:1767319519.173853:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835456 00000100:00100000:3.0:1767319519.173893:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835200 00000100:00100000:3.0:1767319519.173897:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835200 00000100:00100000:3.0:1767319519.173917:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163 00000100:00100000:3.0:1767319519.173923:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982835456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319519.173962:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982835456:12345-0@lo:400:kworker.0 Request processed in 39us (785us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319519.173968:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163 00000100:00100000:3.0:1767319519.173982:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164 00000100:00100000:3.0:1767319519.173987:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982835200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319519.174016:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982835200:12345-0@lo:400:kworker.0 Request processed in 29us (556us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319519.174021:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164 00000100:00100000:3.0:1767319519.174053:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982835328:0@lo:38:kworker.0 00000100:00100000:3.0:1767319519.174079:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319519.174097:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982835584:0@lo:38:kworker.0 00000100:00100000:3.0:1767319519.174116:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319519.174131:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982835712:0@lo:38:kworker.0 00000100:00100000:3.0:1767319519.174154:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319519.174169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982836096:0@lo:38:kworker.0 00000100:00100000:3.0:1767319519.174187:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319519.174211:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835328 00000100:00100000:3.0:1767319519.174216:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835328 00000100:00100000:3.0:1767319519.174232:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835584 00000100:00100000:3.0:1767319519.174235:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835584 00000100:00100000:3.0:1767319519.174257:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836096 00000100:00100000:3.0:1767319519.174259:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836096 00000100:00100000:3.0:1767319519.174312:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1531 00000100:00100000:3.0:1767319519.174317:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982835328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319519.174334:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319519.174369:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982835328:12345-0@lo:38:kworker.0 Request processed in 51us (289us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319519.174376:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1531 00000100:00100000:3.0:1767319519.174387:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1532 00000100:00100000:3.0:1767319519.174391:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982835584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319519.174399:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319519.174420:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982835584:12345-0@lo:38:kworker.0 Request processed in 28us (304us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319519.174425:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1532 00000100:00100000:3.0:1767319519.174434:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1533 00000100:00100000:3.0:1767319519.174438:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982836096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319519.174447:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319519.174468:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982836096:12345-0@lo:38:kworker.0 Request processed in 28us (280us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319519.174473:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1533 00000100:00080000:3.0:1767319519.174508:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319519.174513:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319519.174516:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982835328:0@lo:38:kworker.0 00000100:00080000:3.0:1767319519.174527:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319519.174529:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319519.174531:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982835584:0@lo:38:kworker.0 00000100:00080000:3.0:1767319519.174538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319519.174540:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319519.174543:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982836096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319519.174689:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982835712 00000100:00100000:0.0:1767319519.174693:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982835712 00000100:00100000:0.0:1767319519.174723:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 408 00000100:00100000:0.0:1767319519.174728:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982835712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319519.174736:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319519.174777:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982835712:12345-0@lo:38:kworker.0 Request processed in 48us (622us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319519.174783:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 408 00000100:00080000:0.0:1767319519.174797:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319519.174818:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163982835200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319535 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319519.174829:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982835200:0@lo:400:kworker.0 00010000:00080000:0.0:1767319519.174852:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5f80 x1853163982835456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319535 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319519.174862:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982835456:0@lo:400:kworker.0 00000100:00080000:3.0:1767319519.175006:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319519.175010:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319519.175013:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982835712:0@lo:38:kworker.0 00010000:00010000:3.0:1767319519.364666:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319520.389595:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319520.708875:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982836224:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319520.708946:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319520.709315:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319520.709328:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982836352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319520.709359:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319520.709548:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836224 00000100:00100000:3.0:1767319520.709557:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836224 00000100:00100000:3.0:1767319520.709608:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836352 00000100:00100000:3.0:1767319520.709611:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836352 00000100:00100000:3.0:1767319520.709627:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 790 00000100:00100000:3.0:1767319520.709635:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982836224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319520.709663:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319520.709718:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982836224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (772us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319520.709726:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 790 00000100:00100000:3.0:1767319520.709742:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 791 00000100:00100000:3.0:1767319520.709749:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982836352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319520.709764:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319520.709792:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982836352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (432us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319520.709801:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 791 00000100:00100000:0.0:1767319520.709963:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982836224:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319520.710039:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982836352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319521.175518:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319521.175584:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629568 00000100:00100000:2.0:1767319521.175595:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629568 00000100:00100000:1.0:1767319521.175600:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319521.175629:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629696 00000100:00100000:2.0:1767319521.175632:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629696 00000100:00100000:3.0:1767319521.175733:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1534 00000100:00100000:3.0:1767319521.175742:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915629568:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319521.175764:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319521.175796:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1165 00000100:00100000:2.0:1767319521.175804:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915629696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319521.175942:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915629568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 197us (429us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767319521.175959:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319521.175963:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1534 00000100:00100000:2.0:1767319521.176156:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915629696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 351us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319521.176163:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1165 00010000:00010000:1.0:1767319521.414318:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319522.436571:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319523.460500:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319524.292435:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319524.292449:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319524.292523:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319524.292528:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319524.292532:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319524.292543:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4767 00000100:00080000:3.0:1767319524.292549:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319524.292557:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319524.292585:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982836608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319524.292630:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319524.292634:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319524.292636:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319524.292640:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4767 00000100:00080000:3.0:1767319524.292643:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319524.292648:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319524.292665:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323100 x1853163982836864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319524.292673:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319524.292676:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319524.292678:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319524.292682:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4767 00000100:00080000:3.0:1767319524.292685:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319524.292689:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319524.292702:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982836992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319524.292727:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319524.292731:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319524.292733:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319524.292736:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4767 00000100:00080000:3.0:1767319524.292739:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319524.292742:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319524.292752:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982837376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319524.292801:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982836608:0@lo:38:kworker.0 00000100:00100000:3.0:1767319524.292919:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319524.293174:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982836864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319524.293200:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319524.293222:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982836992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319524.293249:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319524.293332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982837376:0@lo:38:kworker.0 00000100:00100000:3.0:1767319524.293356:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319524.293395:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836608 00000100:00100000:3.0:1767319524.293405:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836608 00000100:00100000:3.0:1767319524.293442:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836864 00000100:00100000:3.0:1767319524.293445:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836864 00000100:00100000:3.0:1767319524.293460:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837376 00000100:00100000:3.0:1767319524.293462:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837376 00000100:00100000:3.0:1767319524.293478:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1535 00000100:00100000:3.0:1767319524.293486:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982836608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319524.293512:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319524.293561:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982836608:12345-0@lo:38:kworker.0 Request processed in 73us (640us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319524.293569:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1535 00000100:00100000:3.0:1767319524.293588:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1536 00000100:00100000:3.0:1767319524.293592:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982836864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319524.293601:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319524.293622:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982836864:12345-0@lo:38:kworker.0 Request processed in 28us (421us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319524.293627:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1536 00000100:00100000:3.0:1767319524.293638:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1537 00000100:00100000:3.0:1767319524.293642:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982837376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319524.293650:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319524.293672:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982837376:12345-0@lo:38:kworker.0 Request processed in 29us (315us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319524.293681:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1537 00000100:00100000:3.0:1767319524.293701:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836992 00000100:00100000:3.0:1767319524.293705:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836992 00000100:00100000:3.0:1767319524.293723:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 409 00000100:00100000:3.0:1767319524.293728:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982836992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319524.293737:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319524.293764:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982836992:12345-0@lo:38:kworker.0 Request processed in 35us (514us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319524.293770:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 409 00000100:00080000:3.0:1767319524.293833:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319524.293838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319524.293842:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982836608:0@lo:38:kworker.0 00000100:00080000:3.0:1767319524.293851:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319524.293854:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319524.293857:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982836864:0@lo:38:kworker.0 00000100:00080000:3.0:1767319524.293863:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319524.293866:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319524.293869:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982836992:0@lo:38:kworker.0 00000100:00080000:3.0:1767319524.293876:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319524.293879:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319524.293882:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982837376:0@lo:38:kworker.0 00000100:00100000:1.0:1767319524.294179:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982836480:0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.294221:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319524.294253:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319524.294259:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982837248:0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.294301:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319524.294330:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836480 00000100:00100000:1.0:1767319524.294335:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836480 00000100:00100000:1.0:1767319524.294383:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1166 00000100:00100000:1.0:1767319524.294390:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982836480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.294435:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982836480:12345-0@lo:400:kworker.0 Request processed in 46us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319524.294442:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1166 00000100:00100000:1.0:1767319524.294462:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837248 00000100:00100000:1.0:1767319524.294466:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837248 00000100:00100000:1.0:1767319524.294495:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 729 00000100:00100000:1.0:1767319524.294502:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982837248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.294533:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982837248:12345-0@lo:400:kworker.0 Request processed in 31us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319524.294540:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 729 00000100:00080000:1.0:1767319524.294556:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319524.294560:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319524.294589:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982837248:0@lo:400:kworker.0 00010000:00080000:1.0:1767319524.294601:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320380 x1853163982836480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319540 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319524.294611:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982836480:0@lo:400:kworker.0 00000100:00100000:0.0:1767319524.294738:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982836736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319524.294774:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319524.294793:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319524.294799:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982837120:0@lo:400:kworker.0 00000100:00100000:0.0:1767319524.294820:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319524.294852:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837120 00000100:00100000:0.0:1767319524.294856:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837120 00000100:00100000:0.0:1767319524.294885:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 730 00000100:00100000:0.0:1767319524.294891:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982837120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319524.294924:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982837120:12345-0@lo:400:kworker.0 Request processed in 33us (103us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319524.294931:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 730 00000100:00080000:0.0:1767319524.295135:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319524.295167:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982837120:0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.295234:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982836736 00000100:00100000:1.0:1767319524.295239:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982836736 00000100:00100000:1.0:1767319524.295291:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167 00000100:00100000:1.0:1767319524.295299:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982836736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319524.295334:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982836736:12345-0@lo:400:kworker.0 Request processed in 36us (560us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319524.295341:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167 00000100:00080000:0.0:1767319524.295523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319524.295542:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322a00 x1853163982836736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319540 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319524.295553:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982836736:0@lo:400:kworker.0 00010000:00010000:1.0:1767319524.486543:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319525.508528:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319525.828813:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319525.828845:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982837504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319525.828938:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319525.829111:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837504 00000100:00100000:0.0:1767319525.829135:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837504 00000100:00100000:0.0:1767319525.829210:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 792 00000100:00100000:0.0:1767319525.829222:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982837504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319525.829264:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319525.829337:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982837504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (398us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319525.829346:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 792 00000100:00100000:3.0:1767319525.829680:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982837504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319525.829853:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319525.829865:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982837632:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319525.829901:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319525.830291:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837632 00000100:00100000:2.0:1767319525.830299:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837632 00000100:00100000:2.0:1767319525.830347:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 793 00000100:00100000:2.0:1767319525.830356:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982837632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319525.830394:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319525.830439:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982837632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (536us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319525.830446:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 793 00000100:00100000:3.0:1767319525.830619:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982837632:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319526.285062:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319526.285251:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319526.286257:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629952 00000100:00100000:1.0:1767319526.286294:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629952 00000100:00100000:1.0:1767319526.286432:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915629824 00000100:00100000:1.0:1767319526.286436:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915629824 00000100:00100000:1.0:1767319526.286467:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1168 00000100:00100000:1.0:1767319526.286479:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915629952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319526.286897:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915629952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 283us (1714us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319526.286908:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1168 00000100:00100000:1.0:1767319526.286935:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1538 00000100:00100000:1.0:1767319526.286940:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915629824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319526.286967:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319526.287001:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915629824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (1750us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319526.287008:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1538 00000100:00080000:0.0:1767319526.287972:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319526.538113:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319527.557701:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319528.584124:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319529.412559:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319529.412573:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319529.412651:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319529.412655:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319529.412659:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319529.412669:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4772 00000100:00080000:3.0:1767319529.412675:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319529.412682:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319529.412707:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163982837888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319529.412788:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982837760:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.412875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.412968:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837760 00000100:00100000:2.0:1767319529.412976:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837760 00000100:00100000:2.0:1767319529.413042:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169 00000100:00100000:2.0:1767319529.413062:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982837760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413125:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982837760:12345-0@lo:400:kworker.0 Request processed in 74us (250us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319529.413134:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169 00000100:00080000:2.0:1767319529.413161:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319529.413198:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163982837760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319545 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319529.413213:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982837760:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413260:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982838016:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413310:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.413336:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838016 00000100:00100000:2.0:1767319529.413340:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838016 00000100:00100000:2.0:1767319529.413365:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1170 00000100:00100000:2.0:1767319529.413370:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982838016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413404:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982838016:12345-0@lo:400:kworker.0 Request processed in 34us (94us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319529.413410:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1170 00000100:00080000:2.0:1767319529.413424:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319529.413439:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396f800 x1853163982838016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319545 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319529.413448:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982838016:0@lo:400:kworker.0 00000100:00080000:3.0:1767319529.413516:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319529.413520:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319529.413523:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319529.413527:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4772 00000100:00080000:3.0:1767319529.413531:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319529.413535:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319529.413552:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163982838144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319529.413564:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319529.413567:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319529.413569:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319529.413572:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4772 00000100:00080000:3.0:1767319529.413575:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319529.413578:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319529.413587:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e680 x1853163982838272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319529.413620:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319529.413623:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319529.413625:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319529.413628:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4772 00000100:00080000:3.0:1767319529.413631:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319529.413635:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319529.413643:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163982838656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319529.413704:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982838528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413734:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.413767:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319529.413772:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982838400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413792:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.413822:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838528 00000100:00100000:2.0:1767319529.413826:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838528 00000100:00100000:2.0:1767319529.413858:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838400 00000100:00100000:2.0:1767319529.413862:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838400 00000100:00100000:2.0:1767319529.413887:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00100000:2.0:1767319529.413894:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982838528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413931:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982838528:12345-0@lo:400:kworker.0 Request processed in 37us (195us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319529.413937:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00000100:00100000:2.0:1767319529.413948:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000100:00100000:2.0:1767319529.413954:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982838400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.413981:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982838400:12345-0@lo:400:kworker.0 Request processed in 27us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319529.413986:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00080000:2.0:1767319529.413998:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319529.414001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319529.414015:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982838528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.414033:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982838400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319529.414073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982837888:0@lo:38:kworker.0 00000100:00100000:2.0:1767319529.414100:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.414127:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982838144:0@lo:38:kworker.0 00000100:00100000:2.0:1767319529.414144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.414162:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982838272:0@lo:38:kworker.0 00000100:00100000:2.0:1767319529.414182:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.414201:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982838656:0@lo:38:kworker.0 00000100:00100000:2.0:1767319529.414246:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319529.414295:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982837888 00000100:00100000:2.0:1767319529.414299:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982837888 00000100:00100000:2.0:1767319529.414317:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838144 00000100:00100000:2.0:1767319529.414319:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838144 00000100:00100000:2.0:1767319529.414330:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838656 00000100:00100000:2.0:1767319529.414332:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838656 00000100:00100000:2.0:1767319529.414341:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1539 00000100:00100000:2.0:1767319529.414346:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982837888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319529.414367:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319529.414399:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982837888:12345-0@lo:38:kworker.0 Request processed in 52us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319529.414405:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1539 00000100:00100000:2.0:1767319529.414416:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1540 00000100:00100000:2.0:1767319529.414419:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982838144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319529.414428:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319529.414449:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982838144:12345-0@lo:38:kworker.0 Request processed in 28us (304us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319529.414455:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1540 00000100:00100000:2.0:1767319529.414465:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1541 00000100:00100000:2.0:1767319529.414469:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982838656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319529.414478:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319529.414500:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982838656:12345-0@lo:38:kworker.0 Request processed in 29us (254us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319529.414506:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1541 00000100:00080000:2.0:1767319529.414534:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319529.414540:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319529.414543:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982837888:0@lo:38:kworker.0 00000100:00080000:2.0:1767319529.414550:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319529.414553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319529.414555:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982838144:0@lo:38:kworker.0 00000100:00080000:2.0:1767319529.414561:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319529.414564:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319529.414566:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982838656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319529.415532:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838272 00000100:00100000:0.0:1767319529.415543:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838272 00000100:00100000:0.0:1767319529.415639:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 410 00000100:00100000:0.0:1767319529.415647:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982838272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319529.415662:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319529.415728:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982838272:12345-0@lo:38:kworker.0 Request processed in 80us (1544us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319529.415734:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 410 00000100:00080000:2.0:1767319529.415870:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319529.415875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319529.415879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982838272:0@lo:38:kworker.0 00010000:00010000:1.0:1767319529.604694:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319530.631183:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319530.949002:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982838784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319530.949017:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319530.949045:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982838912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319530.954605:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319530.954905:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838784 00000100:00100000:2.0:1767319530.954922:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838784 00000100:00100000:2.0:1767319530.954989:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 794 00000100:00100000:2.0:1767319530.955000:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982838784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319530.955049:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319530.955124:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982838784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 126us (531us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319530.955132:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 794 00000100:00100000:0.0:1767319530.955186:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982838784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319530.956314:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319530.957445:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982838912 00000100:00100000:3.0:1767319530.957460:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982838912 00000100:00100000:3.0:1767319530.957538:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795 00000100:00100000:3.0:1767319530.957548:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982838912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319530.957591:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319530.957688:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982838912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (1377us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319530.957697:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795 00000100:00100000:1.0:1767319530.957765:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982838912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319531.406898:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319531.406994:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630208 00000100:00100000:3.0:1767319531.407005:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630208 00000100:00100000:3.0:1767319531.407096:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1171 00000100:00100000:3.0:1767319531.407108:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915630208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319531.407211:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915630208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 106us (324us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319531.407220:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1171 00000100:00080000:2.0:1767319531.407509:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319531.407567:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319531.409340:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630080 00000100:00100000:2.0:1767319531.409366:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630080 00000100:00100000:1.0:1767319531.409731:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1542 00000100:00100000:1.0:1767319531.409752:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915630080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319531.409784:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319531.409951:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915630080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 203us (2388us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319531.409959:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1542 00010000:00010000:3.0:1767319531.652522:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319532.676780:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319533.700494:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319534.532897:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319534.532912:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319534.533019:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319534.533024:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319534.533029:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319534.533039:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4777 00000100:00080000:3.0:1767319534.533045:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319534.533054:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319534.533083:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163982839168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319534.533149:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982839168:0@lo:38:kworker.0 00000100:00100000:0.0:1767319534.533245:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319534.533263:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319534.533300:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982839040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319534.533341:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319534.533382:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839168 00000100:00100000:2.0:1767319534.533392:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839168 00000100:00100000:2.0:1767319534.533429:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839040 00000100:00100000:2.0:1767319534.533432:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839040 00000100:00080000:3.0:1767319534.533481:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319534.533485:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319534.533488:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319534.533492:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4777 00000100:00080000:3.0:1767319534.533495:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319534.533499:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319534.533513:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e680 x1853163982839424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319534.533542:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982839424:0@lo:38:kworker.0 00000100:00100000:0.0:1767319534.533568:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319534.534207:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319534.534213:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319534.534215:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319534.534219:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4777 00000100:00080000:3.0:1767319534.534223:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319534.534227:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319534.534247:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163982839552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319534.534318:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319534.534322:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319534.534323:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319534.534327:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4777 00000100:00080000:3.0:1767319534.534330:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319534.534334:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319534.534346:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ce00 x1853163982839936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319534.534415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319534.535305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982839552:0@lo:38:kworker.0 00000100:00100000:0.0:1767319534.535374:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319534.535414:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982839936:0@lo:38:kworker.0 00000100:00100000:0.0:1767319534.535443:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319534.535594:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839424 00000100:00100000:1.0:1767319534.535604:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839424 00000100:00100000:1.0:1767319534.535650:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172 00000100:00100000:1.0:1767319534.535665:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982839040:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.535734:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982839040:12345-0@lo:400:kworker.0 Request processed in 75us (2393us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319534.535741:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172 00000100:00100000:1.0:1767319534.535762:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1543 00000100:00100000:1.0:1767319534.535767:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982839168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319534.535792:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319534.535824:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982839168:12345-0@lo:38:kworker.0 Request processed in 55us (2579us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319534.535831:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1543 00000100:00100000:1.0:1767319534.535843:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1544 00000100:00100000:1.0:1767319534.535847:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982839424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319534.535856:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319534.535878:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982839424:12345-0@lo:38:kworker.0 Request processed in 31us (2309us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319534.535883:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1544 00000100:00100000:1.0:1767319534.535915:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319534.535924:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982839808:0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536065:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319534.536105:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839808 00000100:00100000:1.0:1767319534.536110:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839808 00000100:00100000:1.0:1767319534.536162:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 733 00000100:00100000:1.0:1767319534.536169:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982839808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536206:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982839808:12345-0@lo:400:kworker.0 Request processed in 38us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319534.536212:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 733 00000100:00080000:1.0:1767319534.536234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319534.536295:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982839808:0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536351:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982839680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536632:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319534.536655:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319534.536662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982839296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536691:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839680 00000100:00100000:1.0:1767319534.536696:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839680 00000100:00100000:1.0:1767319534.536721:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00100000:1.0:1767319534.536727:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982839680:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.536759:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982839680:12345-0@lo:400:kworker.0 Request processed in 32us (127us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319534.536764:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00000100:00080000:1.0:1767319534.536776:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319534.536800:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839552 00000100:00100000:1.0:1767319534.536803:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839552 00000100:00100000:1.0:1767319534.536822:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 411 00000100:00100000:1.0:1767319534.536827:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982839552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319534.536840:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319534.536865:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982839552:12345-0@lo:38:kworker.0 Request processed in 37us (1492us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319534.536871:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 411 00010000:00080000:1.0:1767319534.536913:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d880 x1853163982839040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319550 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319534.536924:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982839040:0@lo:400:kworker.0 00000100:00080000:0.0:1767319534.538891:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319534.538906:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319534.538911:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982839424:0@lo:38:kworker.0 00000100:00080000:0.0:1767319534.538926:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319534.538930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319534.538933:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982839168:0@lo:38:kworker.0 00000100:00080000:0.0:1767319534.538984:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319534.538991:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319534.538994:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982839552:0@lo:38:kworker.0 00000100:00100000:2.0:1767319534.539007:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839936 00000100:00100000:2.0:1767319534.539018:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839936 00000100:00100000:2.0:1767319534.539142:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1545 00000100:00100000:2.0:1767319534.539152:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982839936:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319534.539171:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767319534.539177:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319534.539299:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982839936:12345-0@lo:38:kworker.0 Request processed in 147us (3854us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319534.539309:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1545 00000100:00100000:1.0:1767319534.541718:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982839680:0@lo:400:kworker.0 00000100:00100000:1.0:1767319534.543127:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982839296 00000100:00100000:1.0:1767319534.543145:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982839296 00000100:00100000:1.0:1767319534.543253:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173 00000100:00100000:1.0:1767319534.543264:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982839296:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319534.543524:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319534.543591:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982839296:12345-0@lo:400:kworker.0 Request processed in 328us (4421us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319534.543600:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173 00000100:00080000:1.0:1767319534.543668:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319534.543674:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319534.543678:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982839936:0@lo:38:kworker.0 00010000:00080000:2.0:1767319534.543787:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d180 x1853163982839296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319550 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319534.543800:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982839296:0@lo:400:kworker.0 00010000:00010000:3.0:1767319534.724765:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319535.750679:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319536.069337:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982840192:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319536.069349:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319536.069372:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982840064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319536.080905:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319536.081208:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840064 00000100:00100000:3.0:1767319536.081222:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840064 00000100:00100000:3.0:1767319536.081327:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 796 00000100:00100000:3.0:1767319536.081338:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982840064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319536.081392:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319536.081479:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982840064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (586us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319536.081489:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 796 00000100:00080000:3.0:1767319536.081523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319536.081584:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982840064:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319536.082095:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319536.082136:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840192 00000100:00100000:3.0:1767319536.082143:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840192 00000100:00080000:1.0:1767319536.082182:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319536.082189:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 797 00000100:00100000:3.0:1767319536.082198:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982840192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319536.082221:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319536.082303:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982840192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319536.082311:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 797 00000100:00100000:1.0:1767319536.082688:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982840192:0@lo:13:osp-pre-0-1.0 00000400:00000100:0.0:1767319536.528743:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915630336 offset 0 length 520: 4 00000100:00100000:3.0:1767319536.530817:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319536.530941:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630592 00000100:00100000:0.0:1767319536.530957:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630592 00000100:00100000:0.0:1767319536.531116:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1174 00000100:00100000:0.0:1767319536.531126:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915630592:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319536.531218:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915630592:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (424us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319536.531225:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1174 00000100:00080000:1.0:1767319536.535064:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319536.535421:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319536.535531:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630464 00000100:00100000:1.0:1767319536.535550:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630464 00000100:00100000:1.0:1767319536.535615:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1546 00000100:00100000:1.0:1767319536.535625:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915630464:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319536.535653:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319536.535785:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915630464:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 159us (371us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319536.535793:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1546 00010000:00010000:1.0:1767319536.773010:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319537.797351:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319538.821529:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319539.653030:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.653046:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319539.653119:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.653124:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319539.653129:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319539.653139:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4783 00000100:00080000:3.0:1767319539.653145:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319539.653154:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319539.653184:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982840448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319539.653234:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.653238:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319539.653240:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319539.653244:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4783 00000100:00080000:3.0:1767319539.653247:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319539.653251:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319539.653263:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163982840704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319539.653294:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.653297:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319539.653299:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319539.653303:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4783 00000100:00080000:3.0:1767319539.653306:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319539.653309:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319539.653320:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163982840832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319539.653344:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.653347:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319539.653349:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319539.653353:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4783 00000100:00080000:3.0:1767319539.653355:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319539.653359:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319539.653370:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d880 x1853163982841216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319539.653400:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982840576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319539.653569:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319539.653645:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319539.653651:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982840960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319539.653679:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319539.653894:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319539.653927:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319539.653968:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840960 00000100:00100000:1.0:1767319539.653976:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840960 00000100:00100000:1.0:1767319539.654040:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841088 00000100:00100000:1.0:1767319539.654044:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841088 00000100:00100000:1.0:1767319539.654066:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00100000:1.0:1767319539.654075:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982840960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319539.654131:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982840960:12345-0@lo:400:kworker.0 Request processed in 57us (450us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319539.654138:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00100000:1.0:1767319539.654156:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 736 00000100:00100000:1.0:1767319539.654162:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319539.654190:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841088:12345-0@lo:400:kworker.0 Request processed in 29us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319539.654196:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 736 00000100:00080000:1.0:1767319539.654216:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319539.654220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319539.654248:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840576 00000100:00100000:1.0:1767319539.654252:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840576 00000100:00100000:1.0:1767319539.654299:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175 00000100:00100000:1.0:1767319539.654305:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982840576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319539.654335:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982840576:12345-0@lo:400:kworker.0 Request processed in 30us (767us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319539.654342:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175 00000100:00080000:1.0:1767319539.654359:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319539.654636:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982840960:0@lo:400:kworker.0 00010000:00080000:3.0:1767319539.654649:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ce00 x1853163982840576/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319555 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319539.654659:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982840576:0@lo:400:kworker.0 00000100:00100000:2.0:1767319539.654883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982840320:0@lo:400:kworker.0 00000100:00100000:2.0:1767319539.654921:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319539.654965:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840320 00000100:00100000:2.0:1767319539.654969:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840320 00000100:00100000:2.0:1767319539.655004:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176 00000100:00100000:2.0:1767319539.655011:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982840320:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319539.655044:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982840320:12345-0@lo:400:kworker.0 Request processed in 34us (123us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319539.655050:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176 00000100:00080000:2.0:1767319539.655062:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319539.655206:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319514/real 1767319514] req@ffff9346333a0e00 x1853163982833792/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319539 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319539.655225:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346333a0e00 x1853163982833792/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319539 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319539.655239:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982840448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319539.655264:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319539.655419:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840448 00000100:00100000:2.0:1767319539.655424:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840448 00000100:00100000:2.0:1767319539.655449:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1547 00000100:00100000:2.0:1767319539.655454:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982840448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319539.655484:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319539.655524:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982840448:12345-0@lo:38:kworker.0 Request processed in 69us (258us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319539.655531:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1547 00000100:00100000:1.0:1767319539.655932:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841088:0@lo:400:kworker.0 00010000:00080000:2.0:1767319539.656368:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163982840320/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319555 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319539.656381:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982840320:0@lo:400:kworker.0 00000100:00080000:0.0:1767319539.656448:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319539.656454:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319539.656458:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982840448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319539.656466:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982840704:0@lo:38:kworker.0 00000100:00100000:0.0:1767319539.656644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319539.656689:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982840832:0@lo:38:kworker.0 00000100:00100000:0.0:1767319539.657051:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319539.657083:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982841216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319539.657104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319539.657122:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319539.657136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319539.657140:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982833792:0@lo:250:kworker.0 00000100:00100000:2.0:1767319539.657299:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840704 00000100:00100000:2.0:1767319539.657306:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840704 00000100:00100000:2.0:1767319539.657345:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841216 00000100:00100000:2.0:1767319539.657349:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841216 00000100:00100000:2.0:1767319539.657372:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1767319539.657378:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982840704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319539.657398:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319539.657443:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982840704:12345-0@lo:38:kworker.0 Request processed in 63us (799us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319539.657451:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1767319539.657466:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1549 00000100:00100000:2.0:1767319539.657469:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982841216:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319539.657666:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982840832 00000100:00100000:1.0:1767319539.657671:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982840832 00000100:00100000:1.0:1767319539.657904:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 412 00000100:00100000:1.0:1767319539.657910:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982840832:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767319539.657976:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319539.657981:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319539.657984:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982840704:0@lo:38:kworker.0 00010000:00020000:2.0:1767319539.658884:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319539.658961:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982841216:12345-0@lo:38:kworker.0 Request processed in 1487us (1853us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319539.658971:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1549 00000100:00080000:0.0:1767319539.659087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319539.659093:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319539.659097:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982841216:0@lo:38:kworker.0 00000100:00080000:3.0:1767319539.659775:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319539.659782:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319539.659785:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319539.659791:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4762 00000100:00080000:3.0:1767319539.659796:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319539.659803:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319539.659842:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982841344/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319539.660005:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982841344:0@lo:250:kworker.0 00000400:00000100:0.0:1767319539.660046:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982841344 offset 0 length 520: 4 00010000:00020000:1.0:1767319539.665371:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319539.665537:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982840832:12345-0@lo:38:kworker.0 Request processed in 7614us (8474us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319539.665553:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 412 00000100:00080000:0.0:1767319539.665817:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319539.665824:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319539.665829:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982840832:0@lo:38:kworker.0 00010000:00010000:1.0:1767319539.844583:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319540.868550:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319541.190054:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1767319541.190086:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319541.190180:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319541.190230:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319541.190258:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319541.190357:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841600 00000100:00100000:0.0:1767319541.190369:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841600 00000100:00100000:0.0:1767319541.190457:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841472 00000100:00100000:0.0:1767319541.190461:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841472 00000100:00100000:0.0:1767319541.190482:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798 00000100:00100000:0.0:1767319541.190495:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319541.190538:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319541.190555:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 799 00000100:00100000:1.0:1767319541.190565:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319541.190584:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319541.190592:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (412us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319541.190602:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798 00000100:00100000:1.0:1767319541.190615:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982841472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (357us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319541.190621:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 799 00000100:00080000:3.0:1767319541.192165:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319541.192177:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319541.192252:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319541.192309:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982841472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319541.644467:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319541.644516:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319541.644616:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630720 00000100:00100000:3.0:1767319541.644625:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630720 00000100:00100000:0.0:1767319541.644643:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630848 00000100:00100000:0.0:1767319541.644658:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630848 00000100:00100000:3.0:1767319541.644683:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1550 00000100:00100000:3.0:1767319541.644693:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915630720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319541.644717:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319541.644759:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1177 00000100:00100000:0.0:1767319541.644771:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915630848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319541.644791:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915630720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 97us (333us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319541.644798:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1550 00000100:00100000:0.0:1767319541.644856:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915630848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319541.644870:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1177 00000100:00080000:1.0:1767319541.645132:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319541.892710:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319542.916530:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319543.940631:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319544.772782:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319544.772797:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319544.772890:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319544.772895:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319544.772900:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319544.772911:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4788 00000100:00080000:3.0:1767319544.772917:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319544.772926:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319544.772952:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982841856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319544.772985:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319544.772989:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319544.772991:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319544.772994:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4788 00000100:00080000:3.0:1767319544.772997:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319544.773002:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319544.773013:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e680 x1853163982842112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319544.773022:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319544.773025:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319544.773027:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319544.773031:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4788 00000100:00080000:3.0:1767319544.773033:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319544.773037:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319544.773048:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ce00 x1853163982842240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319544.773092:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319544.773096:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319544.773097:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319544.773101:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4788 00000100:00080000:3.0:1767319544.773103:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319544.773107:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319544.773119:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982842624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319544.773157:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982842368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319544.773253:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319544.773313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982842496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319544.773369:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842368 00000100:00100000:3.0:1767319544.773380:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842368 00000100:00100000:2.0:1767319544.773413:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319544.773491:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842496 00000100:00100000:2.0:1767319544.773498:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842496 00000100:00100000:0.0:1767319544.773581:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982841728:0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.773638:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319544.773680:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319544.773686:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982841984:0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.773735:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319544.773778:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841728 00000100:00100000:0.0:1767319544.773783:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841728 00000100:00100000:0.0:1767319544.773818:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841984 00000100:00100000:0.0:1767319544.773821:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841984 00000100:00100000:0.0:1767319544.773849:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1178 00000100:00100000:0.0:1767319544.773858:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982841728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.773914:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982841728:12345-0@lo:400:kworker.0 Request processed in 57us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319544.773922:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1178 00000100:00100000:0.0:1767319544.773939:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179 00000100:00100000:0.0:1767319544.773945:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982841984:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.773968:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982841984:12345-0@lo:400:kworker.0 Request processed in 23us (233us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319544.773975:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179 00010000:00080000:0.0:1767319544.774195:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163982841984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319560 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319544.774211:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982841984:0@lo:400:kworker.0 00010000:00080000:0.0:1767319544.774227:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396dc00 x1853163982841728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319560 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319544.774234:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982841728:0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.774301:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982841856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319544.774331:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319544.774350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982842112:0@lo:38:kworker.0 00000100:00100000:0.0:1767319544.774369:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319544.774384:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982842240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319544.774427:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319544.774590:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982842624:0@lo:38:kworker.0 00000100:00080000:1.0:1767319544.774744:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319544.774751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319544.774754:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319544.774780:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842240 00000100:00100000:1.0:1767319544.774785:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842240 00000100:00100000:1.0:1767319544.774815:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 413 00000100:00100000:1.0:1767319544.774821:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982842240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319544.774843:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319544.775712:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 737 00000100:00100000:2.0:1767319544.775737:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982842368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319544.783787:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00100000:3.0:1767319544.783839:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982842496:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319544.784006:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982842240:12345-0@lo:38:kworker.0 Request processed in 9180us (9574us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319544.784019:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 413 00000100:00100000:3.0:1767319544.784062:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982842496:12345-0@lo:400:kworker.0 Request processed in 255us (10662us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319544.784069:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00000100:00100000:3.0:1767319544.784102:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982841856 00000100:00100000:3.0:1767319544.784112:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982841856 00000100:00100000:3.0:1767319544.784145:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842112 00000100:00100000:3.0:1767319544.784149:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842112 00000100:00100000:1.0:1767319544.784156:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1551 00000100:00100000:1.0:1767319544.784161:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982841856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319544.784186:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319544.784223:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982841856:12345-0@lo:38:kworker.0 Request processed in 60us (9892us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319544.784561:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319544.784626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319544.784632:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319544.784637:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982841856:0@lo:38:kworker.0 00000100:00080000:0.0:1767319544.784651:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319544.784655:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319544.784658:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982842240:0@lo:38:kworker.0 00000100:00100000:0.0:1767319544.784702:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842624 00000100:00100000:0.0:1767319544.784707:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842624 00000100:00100000:2.0:1767319544.788137:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982842368:12345-0@lo:400:kworker.0 Request processed in 12401us (14873us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319544.788153:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 737 00000100:00100000:2.0:1767319544.788231:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982842496:0@lo:400:kworker.0 00000100:00100000:1.0:1767319544.788354:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1551 00000100:00100000:1.0:1767319544.788405:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1552 00000100:00100000:1.0:1767319544.788415:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982842112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319544.788442:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319544.788536:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982842112:12345-0@lo:38:kworker.0 Request processed in 121us (14165us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319544.788543:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1552 00000100:00080000:1.0:1767319544.788616:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319544.788622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319544.788626:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982842112:0@lo:38:kworker.0 00000100:00100000:3.0:1767319544.788740:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982842368:0@lo:400:kworker.0 00000100:00100000:0.0:1767319544.792296:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1553 00000100:00100000:0.0:1767319544.792313:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982842624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319544.792346:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319544.793887:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982842624:12345-0@lo:38:kworker.0 Request processed in 1562us (9314us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319544.793905:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1553 00000100:00080000:1.0:1767319544.794072:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319544.794080:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319544.794084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982842624:0@lo:38:kworker.0 00010000:00010000:1.0:1767319544.968822:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319545.990375:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319546.309817:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982842752:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319546.309899:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319546.310454:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842752 00000100:00100000:3.0:1767319546.310464:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842752 00000100:00100000:3.0:1767319546.310598:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800 00000100:00100000:3.0:1767319546.310715:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982842752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319546.310761:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319546.310813:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982842752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 206us (914us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319546.310821:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800 00000100:00100000:2.0:1767319546.311548:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982842752:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319546.311573:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319546.311581:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982842880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319546.311613:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319546.311691:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982842880 00000100:00100000:1.0:1767319546.311698:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982842880 00000100:00080000:0.0:1767319546.312012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319546.312017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319546.314755:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 801 00000100:00100000:1.0:1767319546.314773:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982842880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319546.314809:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319546.314914:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982842880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 142us (3292us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319546.314922:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 801 00000100:00100000:2.0:1767319546.314965:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982842880:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319546.764251:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319546.764357:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319546.764442:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915630976 00000100:00100000:3.0:1767319546.764457:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915630976 00000100:00100000:1.0:1767319546.764468:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631104 00000100:00100000:1.0:1767319546.764481:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631104 00000100:00100000:3.0:1767319546.764523:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1554 00000100:00100000:3.0:1767319546.764532:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915630976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319546.764559:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319546.764568:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1180 00000100:00100000:1.0:1767319546.764581:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915631104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319546.764652:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915630976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 117us (409us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319546.764660:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915631104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (298us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319546.764679:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1554 00000100:00080000:2.0:1767319546.764773:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319546.766113:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1180 00010000:00010000:3.0:1767319547.012873:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319548.038165:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319549.060593:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319549.896588:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319549.896604:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319549.897058:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982843008:0@lo:400:kworker.0 00000100:00080000:3.0:1767319549.897099:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319549.897105:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319549.897110:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319549.897117:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4793 00000100:00080000:3.0:1767319549.897123:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319549.897132:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319549.897161:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319549.897166:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982843136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319549.897229:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982843136:0@lo:38:kworker.0 00000100:00100000:2.0:1767319549.897266:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319549.897334:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843008 00000100:00100000:1.0:1767319549.897343:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843008 00000100:00100000:1.0:1767319549.897416:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843136 00000100:00100000:1.0:1767319549.897419:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843136 00000100:00100000:1.0:1767319549.897440:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181 00000100:00100000:1.0:1767319549.897452:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982843008:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319549.897458:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319549.897464:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319549.897466:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319549.897471:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4793 00000100:00080000:3.0:1767319549.897474:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319549.897480:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319549.897497:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982843392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319549.897510:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982843008:12345-0@lo:400:kworker.0 Request processed in 59us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319549.897519:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181 00000100:00100000:0.0:1767319549.897553:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1555 00000100:00100000:2.0:1767319549.897579:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982843392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319549.897607:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319549.897654:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843392 00000100:00100000:2.0:1767319549.897659:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843392 00000100:00080000:1.0:1767319549.897683:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319549.897716:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323100 x1853163982843008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319565 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319549.897730:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982843008:0@lo:400:kworker.0 00000100:00100000:1.0:1767319549.897749:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319549.897755:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982843264:0@lo:400:kworker.0 00000100:00100000:1.0:1767319549.897776:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319549.897795:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843264 00000100:00100000:1.0:1767319549.897798:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843264 00000100:00100000:0.0:1767319549.897816:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982843136:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319549.897818:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1182 00000100:00100000:1.0:1767319549.897826:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982843264:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767319549.897846:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319549.897857:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982843264:12345-0@lo:400:kworker.0 Request processed in 30us (78us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319549.897866:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1182 00000100:00080000:1.0:1767319549.897877:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319549.897893:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320000 x1853163982843264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319565 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319549.897903:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982843264:0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.897920:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982843136:12345-0@lo:38:kworker.0 Request processed in 355us (651us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319549.897928:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1555 00000100:00100000:2.0:1767319549.898145:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1556 00000100:00100000:2.0:1767319549.898151:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982843392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319549.898161:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319549.898191:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982843392:12345-0@lo:38:kworker.0 Request processed in 40us (583us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319549.898198:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1556 00000100:00080000:2.0:1767319549.898226:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319549.898231:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319549.898235:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982843392:0@lo:38:kworker.0 00000100:00080000:2.0:1767319549.898244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319549.898247:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319549.898250:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982843136:0@lo:38:kworker.0 00000100:00080000:3.0:1767319549.898527:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319549.898532:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319549.898535:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319549.898539:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4793 00000100:00080000:3.0:1767319549.898542:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319549.898546:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319549.898565:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322a00 x1853163982843520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319549.898800:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982843520:0@lo:38:kworker.0 00000100:00100000:2.0:1767319549.898836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319549.898934:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843520 00000100:00100000:1.0:1767319549.898940:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843520 00000100:00100000:1.0:1767319549.898969:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 414 00000100:00100000:1.0:1767319549.898975:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982843520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319549.898986:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319549.899065:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319549.899069:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319549.899071:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319549.899077:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4793 00000100:00080000:3.0:1767319549.899079:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319549.899083:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319549.899096:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982843904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319549.899227:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982843776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.899263:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319549.899307:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319549.899313:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982843648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.899349:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319549.899392:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843776 00000100:00100000:0.0:1767319549.899396:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843776 00000100:00100000:0.0:1767319549.899435:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843648 00000100:00100000:0.0:1767319549.899438:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843648 00000100:00100000:0.0:1767319549.899460:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 739 00000100:00100000:0.0:1767319549.899468:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982843776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.899507:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982843776:12345-0@lo:400:kworker.0 Request processed in 40us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319549.899514:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 739 00000100:00100000:0.0:1767319549.899525:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 740 00000100:00100000:0.0:1767319549.899531:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982843648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.899554:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982843648:12345-0@lo:400:kworker.0 Request processed in 24us (205us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319549.899560:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 740 00000100:00100000:0.0:1767319549.899591:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982843648:0@lo:400:kworker.0 00000100:00100000:0.0:1767319549.899600:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982843776:0@lo:400:kworker.0 00000100:00080000:0.0:1767319549.899647:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319549.899650:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319549.900186:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982843904:0@lo:38:kworker.0 00000100:00100000:2.0:1767319549.900222:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319549.900367:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982843904 00000100:00100000:0.0:1767319549.900373:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982843904 00000100:00100000:0.0:1767319549.900400:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1557 00000100:00100000:0.0:1767319549.900405:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982843904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319549.900421:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319549.900455:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982843904:12345-0@lo:38:kworker.0 Request processed in 47us (231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319549.900461:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1557 00000100:00100000:1.0:1767319549.900816:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982843520:12345-0@lo:38:kworker.0 Request processed in 1836us (1975us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319549.900825:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 414 00000100:00080000:2.0:1767319549.901027:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319549.901032:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319549.901036:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982843520:0@lo:38:kworker.0 00000100:00080000:2.0:1767319549.901055:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319549.901057:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319549.901060:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982843904:0@lo:38:kworker.0 00010000:00010000:3.0:1767319550.085265:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319551.108867:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319551.428818:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982844032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319551.428937:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319551.429376:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844032 00000100:00100000:1.0:1767319551.429388:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844032 00000100:00100000:1.0:1767319551.429494:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 802 00000100:00100000:1.0:1767319551.429506:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982844032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319551.429545:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319551.429610:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982844032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 107us (673us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319551.429618:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 802 00000100:00100000:1.0:1767319551.429689:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982844032:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319551.431007:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319551.432139:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982844160:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319551.432242:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319551.432566:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844160 00000100:00100000:3.0:1767319551.432577:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844160 00000100:00100000:3.0:1767319551.432665:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 803 00000100:00100000:3.0:1767319551.432675:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982844160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319551.432711:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319551.432766:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982844160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319551.432774:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 803 00000100:00080000:2.0:1767319551.433161:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319551.433222:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982844160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319551.883497:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319551.883744:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631360 00000100:00100000:0.0:1767319551.883759:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631360 00000100:00100000:0.0:1767319551.883866:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1183 00000100:00100000:0.0:1767319551.883879:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915631360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319551.884433:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319551.885186:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915631360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1305us (1698us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319551.885201:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1183 00000100:00100000:0.0:1767319551.885236:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631232 00000100:00100000:0.0:1767319551.885243:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631232 00000100:00100000:0.0:1767319551.885292:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1558 00000100:00100000:0.0:1767319551.885300:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915631232:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319551.885326:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319551.885362:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915631232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (931us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319551.885369:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1558 00000100:00080000:2.0:1767319551.886034:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319552.138024:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319553.157729:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319554.181884:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319555.012638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319555.012651:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319555.012720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319555.012724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319555.012729:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319555.012738:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4798 00000100:00080000:1.0:1767319555.012745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319555.012749:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767319555.012756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319555.012772:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982844288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319555.012788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4000 x1853163982844416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319555.012841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319555.012845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319555.012848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319555.012853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4798 00000100:00080000:1.0:1767319555.012856:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319555.012864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319555.012874:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319555.012876:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319555.012892:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982844544:0@lo:400:kworker.0 00000100:00100000:1.0:1767319555.012911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2300 x1853163982844672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319555.012923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319555.012927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319555.012930:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319555.012938:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4798 00000100:00080000:1.0:1767319555.012941:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319555.012947:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319555.012956:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319555.012972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3480 x1853163982844800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319555.012999:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844288 00000100:00100000:2.0:1767319555.013010:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844288 00000100:00100000:2.0:1767319555.013081:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184 00000100:00100000:2.0:1767319555.013092:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982844288:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319555.013154:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982844288:12345-0@lo:400:kworker.0 Request processed in 64us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319555.013225:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844544 00000100:00100000:0.0:1767319555.013230:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844544 00000100:00100000:0.0:1767319555.013302:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1185 00000100:00100000:0.0:1767319555.013310:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982844544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319555.013426:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982844416:0@lo:38:kworker.0 00000100:00100000:3.0:1767319555.013499:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319555.013550:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982844672:0@lo:38:kworker.0 00000100:00100000:3.0:1767319555.013573:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319555.013637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319555.013682:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6a00 x1853163982844288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319571 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319555.013700:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982844288:0@lo:400:kworker.0 00000100:00100000:0.0:1767319555.013708:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982844544:12345-0@lo:400:kworker.0 Request processed in 395us (754us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319555.013719:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1185 00000100:00100000:0.0:1767319555.013743:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844416 00000100:00100000:0.0:1767319555.013747:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844416 00000100:00100000:3.0:1767319555.013758:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844672 00000100:00100000:3.0:1767319555.013766:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844672 00000100:00100000:0.0:1767319555.013770:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1559 00000100:00100000:0.0:1767319555.013776:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982844416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319555.013800:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319555.013817:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1560 00000100:00100000:3.0:1767319555.013826:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982844672:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319555.013845:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982844416:12345-0@lo:38:kworker.0 Request processed in 66us (345us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767319555.013849:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319555.013859:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1559 00000100:00100000:3.0:1767319555.013906:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982844672:12345-0@lo:38:kworker.0 Request processed in 79us (331us total) trans 0 rc -19/-19 00010000:00080000:0.0:1767319555.013920:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5f80 x1853163982844544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319571 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319555.013921:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1560 00000100:00100000:0.0:1767319555.013941:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982844544:0@lo:400:kworker.0 00000100:00080000:0.0:1767319555.014098:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319555.014104:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319555.014108:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982844416:0@lo:38:kworker.0 00000100:00080000:0.0:1767319555.014118:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319555.014122:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319555.014124:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982844672:0@lo:38:kworker.0 00000100:00100000:2.0:1767319555.014637:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184 00000100:00080000:1.0:1767319555.015351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319555.015355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319555.015360:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319555.015366:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4798 00000100:00080000:1.0:1767319555.015369:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319555.015376:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319555.015400:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982844928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319555.015408:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2a00 x1853163982845184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319555.015467:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982845056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319555.015479:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319555.015626:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982844800:0@lo:38:kworker.0 00000100:00100000:3.0:1767319555.015669:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844928 00000100:00100000:3.0:1767319555.015678:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844928 00000100:00100000:3.0:1767319555.015733:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 741 00000100:00100000:3.0:1767319555.015742:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982844928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319555.015913:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319555.015970:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845056 00000100:00100000:1.0:1767319555.015987:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845056 00000100:00100000:1.0:1767319555.016015:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 742 00000100:00100000:1.0:1767319555.016022:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982845056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319555.016055:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982845056:12345-0@lo:400:kworker.0 Request processed in 34us (142us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319555.016061:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 742 00000100:00100000:1.0:1767319555.016098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982845056:0@lo:400:kworker.0 00000100:00080000:1.0:1767319555.016139:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319555.016143:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319555.020828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319555.020922:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982845184:0@lo:38:kworker.0 00000100:00100000:0.0:1767319555.021002:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319555.021075:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845184 00000100:00100000:2.0:1767319555.021087:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845184 00000100:00100000:2.0:1767319555.021136:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1561 00000100:00100000:2.0:1767319555.021143:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319555.021174:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319555.021219:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845184:12345-0@lo:38:kworker.0 Request processed in 77us (218us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319555.021227:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1561 00000100:00100000:1.0:1767319555.021741:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982844800 00000100:00100000:1.0:1767319555.021758:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982844800 00000100:00100000:1.0:1767319555.021830:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 415 00000100:00100000:1.0:1767319555.021840:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982844800:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319555.021864:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319555.021949:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982844800:12345-0@lo:38:kworker.0 Request processed in 109us (1132us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319555.021956:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 415 00000100:00080000:0.0:1767319555.022384:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319555.022396:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319555.022401:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982844800:0@lo:38:kworker.0 00000100:00080000:0.0:1767319555.022418:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319555.022421:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319555.022424:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982845184:0@lo:38:kworker.0 00000100:00100000:3.0:1767319555.022531:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982844928:12345-0@lo:400:kworker.0 Request processed in 6785us (7050us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319555.022544:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 741 00000100:00100000:2.0:1767319555.022632:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982844928:0@lo:400:kworker.0 00010000:00010000:1.0:1767319555.205411:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319556.229192:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319556.548579:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982845440:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319556.548631:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982845312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319556.548659:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319556.548693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319556.548774:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845440 00000100:00100000:1.0:1767319556.548783:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845440 00000100:00100000:1.0:1767319556.548832:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845312 00000100:00100000:1.0:1767319556.548835:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845312 00000100:00100000:1.0:1767319556.548849:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804 00000100:00100000:1.0:1767319556.548868:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982845440:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319556.548874:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 805 00000100:00100000:2.0:1767319556.548885:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982845312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319556.548899:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:2.0:1767319556.548917:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319556.548953:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982845440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319556.548965:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982845312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319556.548967:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804 00000100:00100000:2.0:1767319556.548977:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 805 00000100:00100000:1.0:1767319556.549018:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982845440:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319556.553483:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982845312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319557.003800:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319557.003927:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631616 00000100:00100000:1.0:1767319557.003940:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631616 00000100:00100000:1.0:1767319557.004030:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1186 00000100:00100000:1.0:1767319557.004043:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915631616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319557.004121:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915631616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319557.004129:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1186 00000100:00080000:3.0:1767319557.004254:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319557.005116:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319557.005184:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631488 00000100:00100000:1.0:1767319557.005190:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631488 00000100:00100000:1.0:1767319557.005222:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1562 00000100:00100000:1.0:1767319557.005228:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915631488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319557.005250:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319557.005337:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915631488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 107us (223us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319557.005345:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1562 00010000:00010000:1.0:1767319557.253642:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319558.279735:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319559.300495:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319560.132526:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319560.132538:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319560.132602:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319560.132606:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319560.132610:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319560.132617:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4803 00000100:00080000:0.0:1767319560.132622:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319560.132629:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319560.132650:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596300 x1853163982845696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319560.132669:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319560.132672:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319560.132674:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319560.132677:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4803 00000100:00080000:0.0:1767319560.132679:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319560.132682:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319560.132691:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163982845952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319560.132699:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319560.132703:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319560.132704:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319560.132708:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4803 00000100:00080000:0.0:1767319560.132711:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319560.132715:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319560.132730:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594700 x1853163982846080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319560.132735:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982845824:0@lo:400:kworker.0 00000100:00100000:3.0:1767319560.132802:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319560.132865:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982845568:0@lo:400:kworker.0 00000100:00100000:1.0:1767319560.132900:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319560.133028:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845824 00000100:00100000:3.0:1767319560.133036:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845824 00000100:00100000:3.0:1767319560.133099:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845568 00000100:00100000:3.0:1767319560.133102:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845568 00000100:00100000:3.0:1767319560.133128:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187 00000100:00100000:3.0:1767319560.133136:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982845824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319560.133188:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982845824:12345-0@lo:400:kworker.0 Request processed in 52us (384us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319560.133195:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187 00000100:00100000:3.0:1767319560.133210:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188 00000100:00100000:3.0:1767319560.133216:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982845568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319560.133243:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982845568:12345-0@lo:400:kworker.0 Request processed in 28us (342us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319560.133248:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188 00010000:00080000:3.0:1767319560.133315:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597480 x1853163982845824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319576 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319560.133329:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982845824:0@lo:400:kworker.0 00000100:00080000:1.0:1767319560.133395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319560.133417:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638595880 x1853163982845568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319576 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319560.133427:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982845568:0@lo:400:kworker.0 00000100:00100000:1.0:1767319560.133567:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319560.133962:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982845696:0@lo:38:kworker.0 00000100:00100000:2.0:1767319560.134009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319560.134046:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982845952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319560.134110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319560.134133:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982846080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319560.134158:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319560.134194:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845696 00000100:00100000:2.0:1767319560.134198:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845696 00000100:00100000:2.0:1767319560.134217:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982845952 00000100:00100000:2.0:1767319560.134220:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982845952 00000100:00100000:2.0:1767319560.134239:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1563 00000100:00100000:2.0:1767319560.134245:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845696:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319560.134265:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319560.134339:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845696:12345-0@lo:38:kworker.0 Request processed in 92us (328us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319560.134347:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1563 00000100:00100000:2.0:1767319560.134361:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1564 00000100:00100000:2.0:1767319560.134365:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319560.134374:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319560.134396:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982845952:12345-0@lo:38:kworker.0 Request processed in 30us (286us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319560.134401:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1564 00000100:00100000:2.0:1767319560.134420:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319560.134428:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982846336:0@lo:400:kworker.0 00000100:00100000:2.0:1767319560.134450:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319560.134479:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319560.134494:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319560.134498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982845696:0@lo:38:kworker.0 00000100:00080000:2.0:1767319560.134507:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319560.134510:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319560.134512:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982845952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319560.134542:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846080 00000100:00100000:2.0:1767319560.134545:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846080 00000100:00100000:2.0:1767319560.134564:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 416 00000100:00100000:2.0:1767319560.134568:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982846080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319560.134577:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319560.134604:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982846080:12345-0@lo:38:kworker.0 Request processed in 34us (445us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319560.134609:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 416 00000100:00080000:2.0:1767319560.134631:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319560.136881:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846336 00000100:00100000:3.0:1767319560.136894:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846336 00000100:00100000:3.0:1767319560.136966:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00100000:3.0:1767319560.136975:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982846336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319560.137066:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982846336:12345-0@lo:400:kworker.0 Request processed in 92us (2614us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319560.137073:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00100000:3.0:1767319560.137150:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982846336:0@lo:400:kworker.0 00000100:00080000:3.0:1767319560.137183:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319560.138043:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319560.138050:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982846080:0@lo:38:kworker.0 00000100:00100000:1.0:1767319560.145383:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982846208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319560.145469:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319560.145568:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846208 00000100:00100000:3.0:1767319560.145578:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846208 00000100:00100000:3.0:1767319560.145668:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00100000:3.0:1767319560.145677:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982846208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319560.145731:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982846208:12345-0@lo:400:kworker.0 Request processed in 55us (262us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319560.145738:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00000100:00080000:1.0:1767319560.146150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319560.146184:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982846208:0@lo:400:kworker.0 00000100:00080000:0.0:1767319560.151882:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319560.151895:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319560.151900:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319560.151908:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4803 00000100:00080000:0.0:1767319560.151913:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319560.151921:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319560.151967:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ce00 x1853163982846464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319560.152247:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982846464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319560.152345:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319560.152873:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846464 00000100:00100000:0.0:1767319560.152883:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846464 00000100:00100000:0.0:1767319560.152931:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1565 00000100:00100000:0.0:1767319560.152939:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982846464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319560.152961:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319560.153016:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982846464:12345-0@lo:38:kworker.0 Request processed in 77us (670us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319560.153023:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1565 00000100:00080000:2.0:1767319560.153182:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319560.153188:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319560.153192:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982846464:0@lo:38:kworker.0 00010000:00010000:1.0:1767319560.328834:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319561.099733:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915631744 offset 0 length 520: 4 00000100:00100000:0.0:1767319561.100766:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319561.100926:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915631872 00000100:00100000:3.0:1767319561.100937:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915631872 00000100:00100000:3.0:1767319561.100987:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1566 00000100:00100000:3.0:1767319561.100996:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915631872:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319561.101020:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319561.101096:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915631872:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 98us (335us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319561.101103:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1566 00000100:00100000:3.0:1767319561.101382:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319561.101519:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632000 00000100:00100000:0.0:1767319561.101524:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632000 00000100:00100000:0.0:1767319561.101594:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1189 00000100:00100000:0.0:1767319561.101602:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915632000:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319561.101659:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915632000:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 59us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319561.101665:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1189 00000100:00080000:1.0:1767319561.101758:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319561.351526:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319561.668610:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319561.668634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982846592:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319561.668705:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319561.668858:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846592 00000100:00100000:1.0:1767319561.668868:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846592 00000100:00100000:0.0:1767319561.668891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982846720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319561.668927:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319561.668932:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806 00000100:00100000:1.0:1767319561.668944:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982846592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319561.668972:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319561.668996:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846720 00000100:00100000:2.0:1767319561.669002:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846720 00000100:00100000:2.0:1767319561.669120:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807 00000100:00100000:2.0:1767319561.669129:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982846720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319561.669152:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319561.671819:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982846720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2683us (2883us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319561.671835:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807 00000100:00100000:0.0:1767319561.671937:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982846720:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319561.675957:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982846592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 7004us (7240us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319561.675972:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806 00000100:00100000:0.0:1767319561.676006:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982846592:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767319562.372545:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319563.396949:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319564.228393:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319539/real 1767319539] req@ffff934633015f80 x1853163982841344/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319564 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319564.228435:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934633015f80 x1853163982841344/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319564 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319564.228450:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319564.228477:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319564.228482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982841344:0@lo:250:kworker.0 00000100:00080000:3.0:1767319564.228520:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319564.228526:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319564.228530:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319564.228535:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4788 00000100:00080000:3.0:1767319564.228539:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319564.228546:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319564.228577:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163982846848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319564.228645:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319564.228653:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319564.228655:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319564.228659:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4808 00000100:00080000:3.0:1767319564.228661:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319564.228665:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319564.228679:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595880 x1853163982847104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319564.228696:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319564.228699:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319564.228700:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319564.228703:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4808 00000100:00080000:3.0:1767319564.228705:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319564.228709:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319564.228722:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594700 x1853163982847360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319564.228730:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319564.228733:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319564.228735:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319564.228738:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4808 00000100:00080000:3.0:1767319564.228741:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319564.228744:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319564.228778:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb800 x1853163982847488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319564.228809:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319564.228812:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319564.228814:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319564.228818:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4808 00000100:00080000:3.0:1767319564.228820:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319564.228823:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319564.228834:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8e00 x1853163982847872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319564.228860:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982847616:0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.228919:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319564.228972:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319564.228978:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982847744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.229000:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319564.229029:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847616 00000100:00100000:3.0:1767319564.229037:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847616 00000100:00100000:3.0:1767319564.229078:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847744 00000100:00100000:3.0:1767319564.229081:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847744 00000100:00100000:3.0:1767319564.229102:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 745 00000100:00100000:3.0:1767319564.229110:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982847616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.229157:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982847616:12345-0@lo:400:kworker.0 Request processed in 48us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319564.229166:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 745 00000100:00100000:3.0:1767319564.229183:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00100000:3.0:1767319564.229189:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982847744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.229218:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982847744:12345-0@lo:400:kworker.0 Request processed in 29us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319564.229223:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00000100:00100000:3.0:1767319564.229258:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982847744:0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.229296:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982847616:0@lo:400:kworker.0 00000100:00100000:2.0:1767319564.229480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982846848:0@lo:250:kworker.0 00000400:00000100:2.0:1767319564.229507:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982846848 offset 0 length 520: 4 00000100:00100000:2.0:1767319564.229526:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982847104:0@lo:38:kworker.0 00000100:00100000:2.0:1767319564.229549:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319564.229573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982847360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319564.229592:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319564.229620:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982847488:0@lo:38:kworker.0 00000100:00100000:2.0:1767319564.229644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319564.229659:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982847872:0@lo:38:kworker.0 00000100:00100000:2.0:1767319564.229679:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319564.229707:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847104 00000100:00100000:2.0:1767319564.229711:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847104 00000100:00100000:2.0:1767319564.229733:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847360 00000100:00100000:2.0:1767319564.229736:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847360 00000100:00100000:2.0:1767319564.229750:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847872 00000100:00100000:2.0:1767319564.229752:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847872 00000100:00100000:2.0:1767319564.229763:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1567 00000100:00100000:2.0:1767319564.229768:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847104:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319564.229785:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319564.229817:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847104:12345-0@lo:38:kworker.0 Request processed in 48us (266us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319564.229823:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1567 00000100:00100000:2.0:1767319564.229835:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1568 00000100:00100000:2.0:1767319564.229839:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319564.229848:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319564.229871:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847360:12345-0@lo:38:kworker.0 Request processed in 30us (278us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319564.229877:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1568 00000100:00100000:2.0:1767319564.229887:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1569 00000100:00100000:2.0:1767319564.229891:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319564.229899:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319564.229920:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982847872:12345-0@lo:38:kworker.0 Request processed in 27us (240us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319564.229925:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1569 00000100:00100000:2.0:1767319564.229941:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847488 00000100:00100000:2.0:1767319564.229945:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847488 00000100:00100000:2.0:1767319564.229962:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 417 00000100:00100000:2.0:1767319564.229967:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982847488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319564.229975:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319564.229998:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982847488:12345-0@lo:38:kworker.0 Request processed in 30us (354us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319564.230004:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 417 00000100:00080000:2.0:1767319564.230043:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319564.230047:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319564.230051:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982847104:0@lo:38:kworker.0 00000100:00080000:2.0:1767319564.230058:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319564.230060:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319564.230062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982847360:0@lo:38:kworker.0 00000100:00080000:2.0:1767319564.230069:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319564.230072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319564.230074:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982847488:0@lo:38:kworker.0 00000100:00080000:2.0:1767319564.230081:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319564.230083:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319564.230086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982847872:0@lo:38:kworker.0 00000100:00080000:1.0:1767319564.230265:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319564.230296:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319564.230318:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982847232:0@lo:400:kworker.0 00000100:00100000:1.0:1767319564.230348:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319564.230376:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319564.230382:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982846976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319564.230402:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319564.230474:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982847232 00000100:00100000:3.0:1767319564.230478:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982847232 00000100:00100000:3.0:1767319564.230616:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982846976 00000100:00100000:3.0:1767319564.230620:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982846976 00000100:00100000:0.0:1767319564.230663:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1190 00000100:00100000:0.0:1767319564.230673:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982847232:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319564.230716:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982847232:12345-0@lo:400:kworker.0 Request processed in 46us (367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319564.230723:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1190 00010000:00080000:0.0:1767319564.230763:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597480 x1853163982847232/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319564.230773:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982847232:0@lo:400:kworker.0 00000100:00080000:1.0:1767319564.231107:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319564.231230:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191 00000100:00100000:3.0:1767319564.231238:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982846976:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319564.231749:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982846976:12345-0@lo:400:kworker.0 Request processed in 511us (1344us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319564.231757:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191 00010000:00080000:0.0:1767319564.231882:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596300 x1853163982846976/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319564.231893:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982846976:0@lo:400:kworker.0 00010000:00010000:1.0:1767319564.420922:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319565.444617:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319566.222924:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319566.222991:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319566.223096:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632128 00000100:00100000:0.0:1767319566.223106:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632128 00000100:00100000:0.0:1767319566.223164:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632256 00000100:00100000:0.0:1767319566.223168:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632256 00000100:00100000:0.0:1767319566.223218:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1192 00000100:00100000:0.0:1767319566.223228:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915632256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319566.223357:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915632256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (366us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319566.223365:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1192 00000100:00100000:0.0:1767319566.223414:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1570 00000100:00100000:0.0:1767319566.223419:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915632128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319566.223437:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319566.223463:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915632128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (551us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319566.223469:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1570 00000100:00080000:3.0:1767319566.223826:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319566.469304:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319566.788898:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319566.788921:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982848000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319566.788988:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319566.789043:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848000 00000100:00100000:1.0:1767319566.789051:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848000 00000100:00100000:1.0:1767319566.789104:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 808 00000100:00100000:1.0:1767319566.789113:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982848000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319566.789141:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319566.789183:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982848000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (194us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319566.789190:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 808 00000100:00100000:1.0:1767319566.789245:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982848000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319566.790614:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982848128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319566.790671:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319566.790977:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319566.791098:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848128 00000100:00100000:2.0:1767319566.791106:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848128 00000100:00100000:2.0:1767319566.791159:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 809 00000100:00100000:2.0:1767319566.791169:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982848128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319566.791195:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319566.791241:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982848128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (569us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319566.791249:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 809 00000100:00100000:2.0:1767319566.791315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982848128:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319566.791456:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:0.0:1767319567.492458:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319568.521437:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319569.348566:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319569.348577:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319569.348677:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982848256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319569.348752:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319569.348783:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848256 00000100:00100000:2.0:1767319569.348792:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848256 00000100:00080000:0.0:1767319569.348816:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319569.348820:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319569.348824:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319569.348830:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4812 00000100:00080000:0.0:1767319569.348834:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319569.348841:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319569.348859:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193 00000100:00100000:0.0:1767319569.348866:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163982848384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319569.348873:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982848256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319569.348952:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982848512:0@lo:400:kworker.0 00000100:00100000:1.0:1767319569.348976:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319569.349001:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848512 00000100:00100000:1.0:1767319569.349013:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848512 00000100:00100000:1.0:1767319569.349041:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1194 00000100:00100000:1.0:1767319569.349046:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982848512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319569.349080:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982848512:12345-0@lo:400:kworker.0 Request processed in 33us (102us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319569.349088:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1194 00010000:00080000:1.0:1767319569.349121:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396e680 x1853163982848512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319569.349136:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982848512:0@lo:400:kworker.0 00000100:00100000:1.0:1767319569.349162:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982848384:0@lo:38:kworker.0 00000100:00100000:1.0:1767319569.349180:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319569.349194:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848384 00000100:00100000:1.0:1767319569.349197:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848384 00000100:00100000:1.0:1767319569.349208:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1571 00000100:00100000:1.0:1767319569.349211:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982848384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319569.349225:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319569.349254:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982848384:12345-0@lo:38:kworker.0 Request processed in 40us (72us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319569.349260:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1571 00000100:00080000:1.0:1767319569.349306:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319569.349325:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319569.349329:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982848384:0@lo:38:kworker.0 00000100:00080000:1.0:1767319569.349374:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319569.357258:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319569.357284:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319569.357289:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319569.357298:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4812 00000100:00080000:0.0:1767319569.357303:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319569.357310:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319569.357365:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163982848640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319569.357387:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319569.357391:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319569.357393:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319569.357396:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4812 00000100:00080000:0.0:1767319569.357399:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319569.357403:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319569.357416:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e680 x1853163982848768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319569.358424:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319569.358429:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319569.358448:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319569.358453:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4812 00000100:00080000:0.0:1767319569.358456:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319569.358474:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319569.358489:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435ad80 x1853163982849152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319569.358530:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982848896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319569.358600:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319569.358647:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319569.358652:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982849024:0@lo:400:kworker.0 00000100:00100000:0.0:1767319569.358762:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319569.358795:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848896 00000100:00100000:0.0:1767319569.358802:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848896 00000100:00100000:0.0:1767319569.358852:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849024 00000100:00100000:0.0:1767319569.358856:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849024 00000100:00100000:2.0:1767319569.358899:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982848256:12345-0@lo:400:kworker.0 Request processed in 10030us (10145us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319569.358910:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193 00000100:00100000:1.0:1767319569.359033:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982848640:0@lo:38:kworker.0 00000100:00100000:1.0:1767319569.359066:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319569.359090:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982848768:0@lo:38:kworker.0 00000100:00100000:1.0:1767319569.359112:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319569.359129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982849152:0@lo:38:kworker.0 00000100:00100000:1.0:1767319569.359155:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319569.359187:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848768 00000100:00100000:1.0:1767319569.359191:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848768 00000100:00100000:1.0:1767319569.359214:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 418 00000100:00100000:1.0:1767319569.359220:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982848768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319569.359241:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319569.359298:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982848768:12345-0@lo:38:kworker.0 Request processed in 76us (185us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319569.359305:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 418 00000100:00100000:1.0:1767319569.359339:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982848640 00000100:00100000:1.0:1767319569.359343:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982848640 00000100:00100000:1.0:1767319569.359359:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849152 00000100:00100000:1.0:1767319569.359361:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849152 00000100:00100000:1.0:1767319569.359374:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1572 00000100:00100000:1.0:1767319569.359378:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982848640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319569.359388:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319569.359413:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982848640:12345-0@lo:38:kworker.0 Request processed in 34us (347us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319569.359419:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1572 00000100:00100000:1.0:1767319569.359435:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1573 00000100:00100000:1.0:1767319569.359439:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982849152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319569.359447:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319569.359469:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982849152:12345-0@lo:38:kworker.0 Request processed in 29us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319569.359474:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1573 00000100:00080000:1.0:1767319569.359518:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319569.359523:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319569.359526:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982848640:0@lo:38:kworker.0 00000100:00080000:1.0:1767319569.359536:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319569.359539:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319569.359541:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982848768:0@lo:38:kworker.0 00000100:00080000:1.0:1767319569.359552:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319569.359554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319569.359557:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982849152:0@lo:38:kworker.0 00000100:00100000:0.0:1767319569.359634:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000100:00100000:0.0:1767319569.359642:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982848896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319569.359699:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982848896:12345-0@lo:400:kworker.0 Request processed in 59us (1099us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319569.359705:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00000100:00100000:0.0:1767319569.359719:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 748 00000100:00100000:0.0:1767319569.359725:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982849024:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319569.359751:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982849024:12345-0@lo:400:kworker.0 Request processed in 26us (989us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319569.359756:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 748 00000100:00080000:1.0:1767319569.361799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319569.361807:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319569.361868:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982849024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319569.361887:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982848896:0@lo:400:kworker.0 00010000:00080000:3.0:1767319569.362223:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ce00 x1853163982848256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319585 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319569.362232:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982848256:0@lo:400:kworker.0 00010000:00010000:2.0:1767319569.540482:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319570.567615:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319571.340158:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319571.340218:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319571.340434:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632384 00000100:00100000:2.0:1767319571.340463:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632384 00000100:00100000:2.0:1767319571.340538:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632512 00000100:00100000:2.0:1767319571.340542:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632512 00000100:00100000:2.0:1767319571.340610:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1195 00000100:00100000:2.0:1767319571.340622:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915632512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319571.340730:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915632512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 110us (512us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319571.340739:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1195 00000100:00100000:2.0:1767319571.340761:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1574 00000100:00100000:2.0:1767319571.340767:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915632384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319571.340793:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319571.340822:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915632384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 54us (676us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319571.340829:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1574 00000100:00080000:0.0:1767319571.341046:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767319571.588546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319571.909664:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982849280:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319571.909764:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319571.909799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982849408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319571.909847:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319571.912193:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849280 00000100:00100000:1.0:1767319571.912213:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849280 00000100:00100000:1.0:1767319571.912354:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849408 00000100:00100000:1.0:1767319571.912359:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849408 00000100:00100000:1.0:1767319571.912385:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 810 00000100:00100000:1.0:1767319571.912406:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982849280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319571.912451:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319571.912551:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982849280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 157us (2785us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319571.912560:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 810 00000100:00100000:1.0:1767319571.912580:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 811 00000100:00100000:1.0:1767319571.912587:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982849408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319571.912605:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319571.912640:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982849408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (2796us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319571.912646:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 811 00000100:00080000:0.0:1767319571.912872:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319571.912878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319571.913047:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982849408:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319571.913103:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982849280:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319572.614302:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319573.636499:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319574.473808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319574.473819:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319574.473899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319574.473904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319574.473911:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319574.473921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4817 00000100:00080000:1.0:1767319574.473928:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319574.473948:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319574.473978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1500 x1853163982849664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319574.474000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319574.474003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319574.474005:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319574.474010:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4817 00000100:00080000:1.0:1767319574.474012:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319574.474016:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319574.474028:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3480 x1853163982849920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319574.474037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319574.474041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319574.474043:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319574.474047:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4817 00000100:00080000:1.0:1767319574.474049:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319574.474054:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319574.474067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2300 x1853163982850048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319574.474109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319574.474113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319574.474116:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319574.474120:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4817 00000100:00080000:1.0:1767319574.474122:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319574.474127:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319574.474138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e680 x1853163982850432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319574.474180:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982849536:0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.474543:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982850176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.474585:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319574.474638:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319574.474644:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982850304:0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.474667:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319574.474698:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982849792:0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.474747:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850176 00000100:00100000:3.0:1767319574.474757:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850176 00000100:00100000:3.0:1767319574.474824:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850304 00000100:00100000:3.0:1767319574.474827:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850304 00000100:00100000:3.0:1767319574.474850:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 749 00000100:00100000:3.0:1767319574.474859:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982850176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.474920:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982850176:12345-0@lo:400:kworker.0 Request processed in 60us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319574.474928:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 749 00000100:00100000:1.0:1767319574.478352:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319574.478520:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849536 00000100:00100000:1.0:1767319574.478532:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849536 00000100:00100000:1.0:1767319574.478607:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196 00000100:00100000:1.0:1767319574.478621:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982849536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319574.478727:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982849536:12345-0@lo:400:kworker.0 Request processed in 110us (4430us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319574.478735:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196 00010000:00080000:1.0:1767319574.478797:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b0700 x1853163982849536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319590 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319574.478815:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982849536:0@lo:400:kworker.0 00000100:00100000:1.0:1767319574.478881:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00100000:1.0:1767319574.478892:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982850304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319574.478929:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982850304:12345-0@lo:400:kworker.0 Request processed in 40us (4261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319574.478935:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00080000:1.0:1767319574.478957:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319574.478962:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319574.478966:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319574.478999:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982850304:0@lo:400:kworker.0 00000100:00100000:1.0:1767319574.479011:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982850176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319574.479139:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319574.479648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982849664:0@lo:38:kworker.0 00000100:00100000:2.0:1767319574.479702:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319574.479744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982849920:0@lo:38:kworker.0 00000100:00100000:2.0:1767319574.479767:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319574.479790:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982850048:0@lo:38:kworker.0 00000100:00100000:2.0:1767319574.479817:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319574.479842:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982850432:0@lo:38:kworker.0 00000100:00100000:2.0:1767319574.479865:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319574.479907:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849792 00000100:00100000:2.0:1767319574.479914:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849792 00000100:00100000:2.0:1767319574.479970:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849664 00000100:00100000:2.0:1767319574.479974:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849664 00000100:00100000:2.0:1767319574.479984:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197 00000100:00100000:2.0:1767319574.479990:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982849792:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319574.480034:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982849792:12345-0@lo:400:kworker.0 Request processed in 44us (895us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319574.480040:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197 00000100:00100000:2.0:1767319574.480051:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982849920 00000100:00100000:2.0:1767319574.480054:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982849920 00000100:00100000:2.0:1767319574.480061:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850432 00000100:00100000:2.0:1767319574.480063:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850432 00000100:00100000:2.0:1767319574.480071:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1575 00000100:00100000:2.0:1767319574.480075:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982849664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319574.480097:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319574.480126:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982849664:12345-0@lo:38:kworker.0 Request processed in 50us (424us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319574.480132:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1575 00000100:00100000:2.0:1767319574.480141:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1576 00000100:00100000:2.0:1767319574.480145:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982849920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319574.480153:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319574.480175:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982849920:12345-0@lo:38:kworker.0 Request processed in 28us (407us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319574.480181:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1576 00000100:00100000:2.0:1767319574.480191:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1577 00000100:00100000:2.0:1767319574.480195:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982850432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319574.480203:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319574.480237:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982850432:12345-0@lo:38:kworker.0 Request processed in 40us (370us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319574.480244:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1577 00000100:00100000:2.0:1767319574.480262:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850048 00000100:00100000:2.0:1767319574.480266:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850048 00000100:00100000:2.0:1767319574.480331:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 419 00000100:00100000:2.0:1767319574.480335:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982850048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319574.480344:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319574.480370:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982850048:12345-0@lo:38:kworker.0 Request processed in 34us (553us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319574.480376:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 419 00000100:00080000:2.0:1767319574.480405:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319574.480433:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319574.480439:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319574.480443:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982849664:0@lo:38:kworker.0 00000100:00080000:2.0:1767319574.480465:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319574.480470:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319574.480472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982849920:0@lo:38:kworker.0 00000100:00080000:2.0:1767319574.480482:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319574.480485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319574.480488:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982850048:0@lo:38:kworker.0 00000100:00080000:2.0:1767319574.480496:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319574.480500:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319574.480502:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982850432:0@lo:38:kworker.0 00010000:00080000:3.0:1767319574.484858:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2a00 x1853163982849792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319590 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319574.484888:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982849792:0@lo:400:kworker.0 00010000:00010000:3.0:1767319574.660565:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319575.685259:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319576.462040:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319576.462097:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319576.462348:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632768 00000100:00100000:2.0:1767319576.462354:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632640 00000100:00100000:2.0:1767319576.462364:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632640 00000100:00100000:1.0:1767319576.462365:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632768 00000100:00100000:2.0:1767319576.462427:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1578 00000100:00100000:2.0:1767319576.462437:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915632640:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:1.0:1767319576.462463:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1198 00010000:00020000:2.0:1767319576.462470:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319576.462475:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915632768:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319576.462562:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915632640:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 124us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319576.462571:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1578 00000100:00080000:3.0:1767319576.462916:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319576.463611:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915632768:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1130us (1580us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319576.463623:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1198 00010000:00010000:0.0:1767319576.708497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319577.028602:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982850560:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319577.028676:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319577.029318:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319577.029327:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982850688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319577.029362:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319577.037052:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850560 00000100:00100000:0.0:1767319577.037071:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850560 00000100:00100000:3.0:1767319577.037192:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850688 00000100:00100000:3.0:1767319577.037198:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850688 00000100:00080000:2.0:1767319577.037263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319577.037294:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 812 00000100:00100000:3.0:1767319577.037305:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982850688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319577.037340:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319577.037416:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 813 00000100:00100000:3.0:1767319577.037416:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982850688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 111us (8051us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319577.037431:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982850560:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319577.037431:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 812 00002000:00000024:0.0:1767319577.037451:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319577.037491:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982850688:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319577.037492:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982850560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (8813us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319577.037512:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982850560:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319577.039688:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 813 00000100:00080000:2.0:1767319577.039996:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767319577.732650:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319578.756599:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319579.588481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319579.588492:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319579.588550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319579.588555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319579.588558:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319579.588567:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4823 00000100:00080000:3.0:1767319579.588571:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319579.588577:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319579.588599:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb100 x1853163982850944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319579.588612:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982850816:0@lo:400:kworker.0 00000100:00080000:3.0:1767319579.588637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319579.588641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319579.588643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319579.588648:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4823 00000100:00080000:3.0:1767319579.588651:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319579.588655:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319579.588665:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982850944:0@lo:38:kworker.0 00000100:00100000:3.0:1767319579.588670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163982851200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319579.588683:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319579.588683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319579.588688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319579.588691:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319579.588695:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4823 00000100:00080000:3.0:1767319579.588699:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319579.588703:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319579.588716:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163982851328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319579.588721:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850816 00000100:00100000:2.0:1767319579.588730:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850816 00000100:00080000:3.0:1767319579.588758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319579.588763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319579.588765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319579.588768:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319579.588769:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4823 00000100:00080000:3.0:1767319579.588774:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319579.588778:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982851072:0@lo:400:kworker.0 00000100:00080000:3.0:1767319579.588779:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319579.588820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163982851712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319579.588843:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982851456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319579.588876:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319579.588899:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319579.588906:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982851584:0@lo:400:kworker.0 00000100:00100000:3.0:1767319579.588930:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319579.588961:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851456 00000100:00100000:3.0:1767319579.588966:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851456 00000100:00100000:1.0:1767319579.588969:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319579.588974:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851584 00000100:00100000:0.0:1767319579.588979:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851584 00000100:00100000:1.0:1767319579.589013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319579.589031:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982851200:0@lo:38:kworker.0 00000100:00100000:1.0:1767319579.589051:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319579.589068:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982851328:0@lo:38:kworker.0 00000100:00100000:1.0:1767319579.589102:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319579.589122:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982851712:0@lo:38:kworker.0 00000100:00100000:1.0:1767319579.589152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319579.589258:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851072 00000100:00100000:2.0:1767319579.589261:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851072 00000100:00100000:1.0:1767319579.589763:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851328 00000100:00100000:1.0:1767319579.589769:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851328 00000100:00100000:1.0:1767319579.589796:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 420 00000100:00100000:1.0:1767319579.590005:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982851328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319579.590026:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319579.590462:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982851328:12345-0@lo:38:kworker.0 Request processed in 458us (1358us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319579.590468:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 420 00000100:00100000:1.0:1767319579.590498:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982850944 00000100:00100000:1.0:1767319579.590502:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982850944 00000100:00100000:1.0:1767319579.590512:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851200 00000100:00100000:1.0:1767319579.590514:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851200 00000100:00100000:1.0:1767319579.590520:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851712 00000100:00100000:1.0:1767319579.590521:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851712 00000100:00100000:1.0:1767319579.590529:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767319579.590535:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982850816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319579.590569:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982850816:12345-0@lo:400:kworker.0 Request processed in 36us (1890us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319579.590573:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767319579.590582:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1579 00000100:00100000:1.0:1767319579.590584:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982850944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319579.590592:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319579.590606:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982850944:12345-0@lo:38:kworker.0 Request processed in 21us (1594us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319579.590610:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1579 00000100:00100000:1.0:1767319579.590615:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1580 00000100:00100000:1.0:1767319579.590618:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982851200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319579.590623:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319579.590634:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982851200:12345-0@lo:38:kworker.0 Request processed in 16us (1584us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319579.590638:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1580 00000100:00100000:1.0:1767319579.590643:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1581 00000100:00100000:1.0:1767319579.590645:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982851712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319579.590650:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319579.590660:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982851712:12345-0@lo:38:kworker.0 Request processed in 14us (1509us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319579.590665:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1581 00000100:00080000:1.0:1767319579.591268:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319579.591302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319579.591305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982851200:0@lo:38:kworker.0 00000100:00080000:1.0:1767319579.591322:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319579.591326:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319579.591330:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982851328:0@lo:38:kworker.0 00000100:00080000:1.0:1767319579.591339:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319579.591342:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319579.591345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982851712:0@lo:38:kworker.0 00000100:00080000:1.0:1767319579.591355:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319579.591358:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319579.591360:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982850944:0@lo:38:kworker.0 00010000:00080000:1.0:1767319579.591410:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eaa00 x1853163982850816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319595 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319579.591421:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982850816:0@lo:400:kworker.0 00000100:00100000:3.0:1767319579.592120:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 751 00000100:00100000:3.0:1767319579.592143:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982851456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319579.592212:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982851456:12345-0@lo:400:kworker.0 Request processed in 83us (3335us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319579.592220:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 751 00000100:00100000:3.0:1767319579.592262:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982851456:0@lo:400:kworker.0 00000100:00100000:0.0:1767319579.592598:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00100000:0.0:1767319579.592607:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982851584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319579.592687:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982851584:12345-0@lo:400:kworker.0 Request processed in 83us (3758us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319579.592692:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00100000:3.0:1767319579.592752:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982851584:0@lo:400:kworker.0 00000100:00080000:1.0:1767319579.593032:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319579.593039:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319579.593043:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319579.593100:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200 00000100:00100000:2.0:1767319579.593109:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982851072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319579.593167:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982851072:12345-0@lo:400:kworker.0 Request processed in 60us (4200us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319579.593174:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200 00010000:00080000:1.0:1767319579.593208:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9f80 x1853163982851072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319595 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319579.593220:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982851072:0@lo:400:kworker.0 00010000:00010000:0.0:1767319579.780445:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319580.804612:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319581.579949:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319581.580001:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319581.580106:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915632896 00000100:00100000:2.0:1767319581.580118:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915632896 00000100:00100000:2.0:1767319581.580169:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633024 00000100:00100000:2.0:1767319581.580173:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633024 00000100:00100000:2.0:1767319581.580226:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1201 00000100:00100000:2.0:1767319581.580236:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915633024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319581.580342:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915633024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 107us (340us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319581.580351:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1201 00000100:00100000:2.0:1767319581.580368:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1582 00000100:00100000:2.0:1767319581.580372:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915632896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319581.580390:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319581.580415:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915632896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319581.580421:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1582 00000100:00080000:0.0:1767319581.580770:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319581.828891:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319582.149445:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982851840:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319582.149535:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319582.149886:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851840 00000100:00100000:0.0:1767319582.149895:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851840 00000100:00100000:0.0:1767319582.149959:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 814 00000100:00100000:0.0:1767319582.149968:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982851840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319582.149997:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319582.150044:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982851840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (509us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319582.150051:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 814 00000100:00100000:0.0:1767319582.150129:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982851840:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319582.150648:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319582.150878:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982851968:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319582.150911:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319582.151108:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982851968 00000100:00100000:1.0:1767319582.151115:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982851968 00000100:00100000:1.0:1767319582.151158:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 815 00000100:00100000:1.0:1767319582.151164:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982851968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319582.151187:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319582.151233:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982851968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (321us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319582.151238:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 815 00000100:00080000:2.0:1767319582.151289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319582.151385:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982851968:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319582.855592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319583.876527:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319584.708523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319584.708538:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319584.708590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319584.708593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319584.708596:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319584.708602:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4828 00000100:00080000:3.0:1767319584.708606:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319584.708611:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319584.708627:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966300 x1853163982852224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319584.708648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319584.708651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319584.708653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319584.708676:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4828 00000100:00080000:3.0:1767319584.708679:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319584.708683:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319584.708684:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982852224:0@lo:38:kworker.0 00000100:00100000:3.0:1767319584.708694:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965500 x1853163982852480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319584.708703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319584.708706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319584.708707:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319584.708710:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4828 00000100:00080000:3.0:1767319584.708712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319584.708715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319584.708724:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964700 x1853163982852608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319584.708745:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319584.708753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319584.708756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319584.708757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319584.708760:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4828 00000100:00080000:3.0:1767319584.708763:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319584.708765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319584.708774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163982852992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319584.708795:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982852736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319584.708800:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982852480:0@lo:38:kworker.0 00000100:00100000:2.0:1767319584.708820:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319584.708821:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319584.708840:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982852608:0@lo:38:kworker.0 00000100:00100000:3.0:1767319584.708842:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319584.708847:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982852864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319584.708862:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319584.708862:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319584.708880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982852992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319584.708891:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852736 00000100:00100000:3.0:1767319584.708898:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852736 00000100:00100000:2.0:1767319584.708899:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319584.708946:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852864 00000100:00100000:3.0:1767319584.708949:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852864 00000100:00100000:3.0:1767319584.708970:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 753 00000100:00100000:3.0:1767319584.708979:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982852736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319584.709026:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982852736:12345-0@lo:400:kworker.0 Request processed in 47us (202us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319584.709057:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 753 00000100:00100000:2.0:1767319584.709073:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 754 00000100:00100000:2.0:1767319584.709081:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982852864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319584.709113:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982852864:12345-0@lo:400:kworker.0 Request processed in 33us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319584.709115:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982852096:0@lo:400:kworker.0 00000100:00100000:1.0:1767319584.709160:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852224 00000100:00100000:1.0:1767319584.709164:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852224 00000100:00100000:1.0:1767319584.709268:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852480 00000100:00100000:1.0:1767319584.709290:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852480 00000100:00100000:1.0:1767319584.709308:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852992 00000100:00100000:1.0:1767319584.709311:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852992 00000100:00100000:1.0:1767319584.709322:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1583 00000100:00100000:1.0:1767319584.709326:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319584.709342:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319584.709455:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852224:12345-0@lo:38:kworker.0 Request processed in 52us (634us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319584.709462:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1583 00000100:00100000:1.0:1767319584.709476:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1584 00000100:00100000:1.0:1767319584.709481:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319584.709493:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319584.709519:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852480:12345-0@lo:38:kworker.0 Request processed in 37us (698us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319584.709525:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1584 00000100:00100000:1.0:1767319584.709536:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1585 00000100:00100000:1.0:1767319584.709540:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319584.709549:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319584.709572:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982852992:12345-0@lo:38:kworker.0 Request processed in 31us (672us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319584.709578:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1585 00000100:00100000:1.0:1767319584.709596:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852608 00000100:00100000:1.0:1767319584.709600:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852608 00000100:00100000:1.0:1767319584.709619:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 421 00000100:00100000:1.0:1767319584.709623:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982852608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319584.709631:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319584.709655:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982852608:12345-0@lo:38:kworker.0 Request processed in 30us (792us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319584.709660:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 421 00000100:00100000:0.0:1767319584.709732:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319584.709751:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319584.709759:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982852352:0@lo:400:kworker.0 00000100:00100000:0.0:1767319584.709777:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319584.709795:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852096 00000100:00100000:0.0:1767319584.709798:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852096 00000100:00080000:1.0:1767319584.709916:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319584.709921:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319584.709924:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982852480:0@lo:38:kworker.0 00000100:00080000:1.0:1767319584.709934:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319584.709937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319584.709939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982852608:0@lo:38:kworker.0 00000100:00080000:1.0:1767319584.709949:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319584.709951:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319584.709953:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982852992:0@lo:38:kworker.0 00000100:00080000:1.0:1767319584.709960:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319584.709963:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319584.709966:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982852224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319584.710013:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982852352 00000100:00100000:0.0:1767319584.710016:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982852352 00000100:00100000:0.0:1767319584.710046:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1202 00000100:00100000:0.0:1767319584.710052:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982852096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319584.710082:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982852096:12345-0@lo:400:kworker.0 Request processed in 31us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319584.710088:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1202 00000100:00100000:0.0:1767319584.710097:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203 00000100:00100000:0.0:1767319584.710102:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982852352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319584.710120:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982852352:12345-0@lo:400:kworker.0 Request processed in 19us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319584.710125:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203 00010000:00080000:0.0:1767319584.710147:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f967480 x1853163982852352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319584.710157:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982852352:0@lo:400:kworker.0 00010000:00080000:0.0:1767319584.710164:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965880 x1853163982852096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319584.710306:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319584.710310:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319584.710312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319584.710336:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982852864:0@lo:400:kworker.0 00000100:00100000:3.0:1767319584.710345:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982852736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319584.710972:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982852096:0@lo:400:kworker.0 00000100:00100000:2.0:1767319584.711699:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 754 00010000:00010000:2.0:1767319584.900466:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319585.924485:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319586.710360:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319586.710406:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767319586.710424:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915633408 offset 0 length 520: 4 00000100:00100000:1.0:1767319586.710530:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633280 00000100:00100000:1.0:1767319586.710542:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633280 00000100:00100000:0.0:1767319586.710598:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633152 00000100:00100000:0.0:1767319586.710604:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633152 00000100:00100000:1.0:1767319586.710627:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1204 00000100:00100000:0.0:1767319586.710638:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1586 00000100:00100000:1.0:1767319586.710641:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915633280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319586.710649:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915633152:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319586.710671:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319586.710716:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915633280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (366us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319586.710724:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1204 00000100:00080000:3.0:1767319586.710799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319586.714126:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915633152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3467us (3707us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319586.714141:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1586 00010000:00010000:3.0:1767319586.948983:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319587.268508:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982853120:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319587.268570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982853248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319587.268581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319587.268608:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319587.268678:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853120 00000100:00100000:2.0:1767319587.268682:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853248 00000100:00100000:1.0:1767319587.268687:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853120 00000100:00100000:2.0:1767319587.268688:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853248 00000100:00100000:2.0:1767319587.268749:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 816 00000100:00100000:2.0:1767319587.268760:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982853248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319587.268789:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319587.268860:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 817 00000100:00100000:1.0:1767319587.268868:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982853120:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319587.268881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319587.268887:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767319587.268889:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319587.268948:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982853120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (367us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319587.268955:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 817 00000100:00100000:0.0:1767319587.270113:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982853120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319587.272318:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982853248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3548us (3697us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319587.272335:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 816 00000100:00100000:3.0:1767319587.277690:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982853248:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319587.972901:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319588.996521:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319589.828399:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319564/real 1767319564] req@ffff934638595500 x1853163982846848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319589 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319589.828434:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934638595500 x1853163982846848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319589 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319589.828460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319589.828467:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319589.828471:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319589.828490:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319589.828494:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982846848:0@lo:250:kworker.0 00000100:00080000:3.0:1767319589.828529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319589.828534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319589.828537:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319589.828542:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4833 00000100:00080000:3.0:1767319589.828547:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319589.828553:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319589.828573:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965880 x1853163982853504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319589.828606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319589.828610:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319589.828612:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319589.828616:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4833 00000100:00080000:3.0:1767319589.828618:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319589.828622:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319589.828646:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966300 x1853163982853760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319589.828649:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982853632:0@lo:400:kworker.0 00000100:00080000:3.0:1767319589.828663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319589.828667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319589.828668:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319589.828673:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4833 00000100:00080000:3.0:1767319589.828675:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319589.828678:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319589.828692:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163982853888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319589.828714:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319589.828756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319589.828760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319589.828762:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319589.828767:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4833 00000100:00080000:3.0:1767319589.828770:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319589.828773:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767319589.828774:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319589.828785:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982853376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319589.828791:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163982854272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319589.828807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319589.828811:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319589.828814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319589.828817:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319589.828822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4812 00000100:00080000:3.0:1767319589.828824:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319589.828829:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319589.828866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581180 x1853163982854400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319589.828890:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853632 00000100:00100000:0.0:1767319589.828897:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853632 00000100:00100000:3.0:1767319589.828949:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982854016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319589.828968:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853376 00000100:00100000:0.0:1767319589.828973:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853376 00000100:00100000:3.0:1767319589.828985:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319589.829010:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319589.829012:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205 00000100:00100000:3.0:1767319589.829018:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982854144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319589.829025:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982853632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319589.829043:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319589.829084:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854016 00000100:00100000:0.0:1767319589.829086:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982853632:12345-0@lo:400:kworker.0 Request processed in 63us (371us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319589.829089:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854016 00000100:00100000:0.0:1767319589.829098:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205 00000100:00100000:0.0:1767319589.829118:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1206 00000100:00100000:0.0:1767319589.829128:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982853376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319589.829128:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854144 00000100:00100000:3.0:1767319589.829133:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854144 00000100:00100000:3.0:1767319589.829166:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 755 00000100:00100000:0.0:1767319589.829168:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982853376:12345-0@lo:400:kworker.0 Request processed in 41us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319589.829179:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1206 00000100:00100000:3.0:1767319589.829180:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982854016:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319589.829202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319589.829208:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319589.829211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319589.829225:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982854016:12345-0@lo:400:kworker.0 Request processed in 50us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319589.829232:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 755 00010000:00080000:0.0:1767319589.829238:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965c00 x1853163982853376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319605 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319589.829249:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 756 00000100:00100000:0.0:1767319589.829252:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982853376:0@lo:400:kworker.0 00010000:00080000:0.0:1767319589.829265:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f967480 x1853163982853632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319605 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319589.829295:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982854144:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319589.829296:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982853632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319589.829336:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982854144:12345-0@lo:400:kworker.0 Request processed in 66us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319589.829344:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 756 00000100:00100000:3.0:1767319589.829371:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982854144:0@lo:400:kworker.0 00000100:00100000:3.0:1767319589.829382:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982854016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319589.829403:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982853504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319589.829431:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319589.829461:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982853760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319589.829480:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319589.829505:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982853888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319589.829529:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319589.829550:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982854272:0@lo:38:kworker.0 00000100:00100000:1.0:1767319589.829569:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319589.829585:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982854400:0@lo:250:kworker.0 00000400:00000100:1.0:1767319589.829600:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982854400 offset 0 length 520: 4 00000100:00100000:1.0:1767319589.829626:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853888 00000100:00100000:1.0:1767319589.829630:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853888 00000100:00100000:1.0:1767319589.829649:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 422 00000100:00100000:1.0:1767319589.829653:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982853888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319589.829669:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319589.829703:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982853888:12345-0@lo:38:kworker.0 Request processed in 48us (173us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319589.829711:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 422 00000100:00100000:2.0:1767319589.829723:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853504 00000100:00100000:2.0:1767319589.829728:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853504 00000100:00100000:1.0:1767319589.829736:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982853760 00000100:00100000:1.0:1767319589.829741:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982853760 00000100:00100000:2.0:1767319589.829759:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854272 00000100:00100000:1.0:1767319589.829764:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1587 00000100:00100000:2.0:1767319589.829766:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854272 00000100:00100000:1.0:1767319589.829772:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982853504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319589.829783:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319589.829787:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1588 00000100:00100000:2.0:1767319589.829794:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982853760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319589.829807:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319589.829813:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982853504:12345-0@lo:38:kworker.0 Request processed in 40us (379us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319589.829820:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1587 00000100:00100000:1.0:1767319589.829835:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1589 00000100:00100000:1.0:1767319589.829840:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982854272:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319589.829846:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982853760:12345-0@lo:38:kworker.0 Request processed in 50us (363us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767319589.829851:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319589.829856:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1588 00000100:00100000:1.0:1767319589.829881:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982854272:12345-0@lo:38:kworker.0 Request processed in 39us (310us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319589.829887:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1589 00000100:00080000:1.0:1767319589.829921:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319589.829926:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319589.829929:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982853504:0@lo:38:kworker.0 00000100:00080000:1.0:1767319589.829948:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319589.829952:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319589.829955:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982853760:0@lo:38:kworker.0 00000100:00080000:1.0:1767319589.829963:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319589.829966:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319589.829969:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982853888:0@lo:38:kworker.0 00000100:00080000:1.0:1767319589.829976:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319589.829979:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319589.829982:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982854272:0@lo:38:kworker.0 00010000:00010000:3.0:1767319590.021783:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319591.045976:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319591.819832:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319591.819963:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319591.820145:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633664 00000100:00100000:3.0:1767319591.820154:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633664 00000100:00100000:3.0:1767319591.820249:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633536 00000100:00100000:3.0:1767319591.820253:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633536 00000100:00100000:3.0:1767319591.820300:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1207 00000100:00100000:3.0:1767319591.820309:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915633664:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319591.820379:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915633664:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (555us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319591.820387:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1207 00000100:00100000:3.0:1767319591.820419:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1590 00000100:00100000:3.0:1767319591.820423:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915633536:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319591.820443:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319591.820465:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915633536:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (503us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319591.820472:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1590 00000100:00080000:0.0:1767319591.820784:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319592.068739:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319592.388677:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982854528:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319592.388746:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319592.388848:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854528 00000100:00100000:3.0:1767319592.388855:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854528 00000100:00100000:3.0:1767319592.388931:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 818 00000100:00100000:3.0:1767319592.388940:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982854528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319592.388969:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319592.389014:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982854528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319592.389020:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 818 00000100:00100000:3.0:1767319592.389064:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982854528:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319592.389124:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319592.390147:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319592.390161:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982854656:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319592.390198:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319592.392202:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854656 00000100:00100000:2.0:1767319592.392214:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854656 00000100:00100000:2.0:1767319592.392378:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 819 00000100:00100000:2.0:1767319592.392389:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982854656:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319592.392423:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319592.392523:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982854656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 135us (2323us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319592.392531:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 819 00000100:00080000:1.0:1767319592.392840:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319592.392956:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982854656:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767319593.092546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319594.119268:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319594.948623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319594.948634:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319594.948702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319594.948707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319594.948712:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319594.948721:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4838 00000100:00080000:3.0:1767319594.948727:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319594.948732:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982854784:0@lo:400:kworker.0 00000100:00080000:3.0:1767319594.948737:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319594.948774:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581500 x1853163982854912/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319594.948808:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319594.948814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319594.948818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319594.948820:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319594.948825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4838 00000100:00080000:3.0:1767319594.948828:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319594.948833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319594.948856:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582a00 x1853163982855168/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319594.948857:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319594.948867:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982855040:0@lo:400:kworker.0 00000100:00080000:3.0:1767319594.948870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319594.948875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319594.948878:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319594.948882:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4838 00000100:00080000:3.0:1767319594.948885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319594.948889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319594.948895:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319594.948908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583100 x1853163982855296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319594.948930:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854784 00000100:00100000:0.0:1767319594.948937:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854784 00000100:00080000:3.0:1767319594.948949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319594.948954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319594.948956:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319594.948959:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4838 00000100:00080000:3.0:1767319594.948962:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319594.948966:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319594.948981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580000 x1853163982855680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319594.948999:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855040 00000100:00100000:0.0:1767319594.949004:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855040 00000100:00100000:3.0:1767319594.949007:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982855424:0@lo:400:kworker.0 00000100:00100000:0.0:1767319594.949031:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208 00000100:00100000:0.0:1767319594.949041:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982854784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319594.949049:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319594.949078:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319594.949084:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982855552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319594.949113:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982854784:12345-0@lo:400:kworker.0 Request processed in 73us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319594.949118:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319594.949127:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208 00000100:00100000:0.0:1767319594.949146:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209 00000100:00100000:0.0:1767319594.949154:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982855040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319594.949167:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855424 00000100:00100000:3.0:1767319594.949172:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855424 00000100:00100000:0.0:1767319594.949182:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982855040:12345-0@lo:400:kworker.0 Request processed in 29us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319594.949189:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209 00000100:00100000:3.0:1767319594.949205:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855552 00000100:00100000:3.0:1767319594.949210:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855552 00000100:00100000:3.0:1767319594.949233:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 757 00000100:00100000:3.0:1767319594.949240:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982855424:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319594.949296:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982855424:12345-0@lo:400:kworker.0 Request processed in 57us (250us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319594.949304:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 757 00000100:00100000:3.0:1767319594.949322:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 758 00010000:00080000:0.0:1767319594.949327:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580700 x1853163982855040/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319610 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319594.949332:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982855552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319594.949351:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982855040:0@lo:400:kworker.0 00010000:00080000:0.0:1767319594.949366:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583480 x1853163982854784/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319610 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319594.949372:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982855552:12345-0@lo:400:kworker.0 Request processed in 42us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319594.949380:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982854784:0@lo:400:kworker.0 00000100:00100000:3.0:1767319594.949385:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 758 00000100:00100000:3.0:1767319594.949427:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982855552:0@lo:400:kworker.0 00000100:00100000:3.0:1767319594.949437:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982855424:0@lo:400:kworker.0 00000100:00080000:1.0:1767319594.949487:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319594.949491:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319594.949494:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319594.949872:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982854912:0@lo:38:kworker.0 00000100:00100000:2.0:1767319594.949914:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319594.949942:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982855168:0@lo:38:kworker.0 00000100:00100000:2.0:1767319594.949966:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319594.949991:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982855296:0@lo:38:kworker.0 00000100:00100000:1.0:1767319594.949996:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982854912 00000100:00100000:1.0:1767319594.950002:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982854912 00000100:00100000:1.0:1767319594.950037:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855168 00000100:00100000:1.0:1767319594.950041:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855168 00000100:00100000:1.0:1767319594.950055:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1591 00000100:00100000:1.0:1767319594.950061:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982854912:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319594.950067:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767319594.950091:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319594.950092:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982855680:0@lo:38:kworker.0 00000100:00100000:2.0:1767319594.950131:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319594.950137:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982854912:12345-0@lo:38:kworker.0 Request processed in 74us (231us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319594.950146:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1591 00000100:00100000:1.0:1767319594.950160:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855680 00000100:00100000:1.0:1767319594.950163:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855680 00000100:00080000:2.0:1767319594.950167:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319594.950174:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319594.950175:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1592 00000100:00100000:2.0:1767319594.950182:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982854912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319594.950184:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982855168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319594.950209:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319594.950215:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1593 00000100:00100000:2.0:1767319594.950222:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982855680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319594.950234:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319594.950241:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982855168:12345-0@lo:38:kworker.0 Request processed in 56us (275us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319594.950248:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1592 00000100:00100000:1.0:1767319594.950268:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855296 00000100:00100000:1.0:1767319594.950289:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855296 00000100:00100000:2.0:1767319594.950297:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982855680:12345-0@lo:38:kworker.0 Request processed in 46us (139us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319594.950306:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1593 00000100:00100000:1.0:1767319594.950310:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 423 00000100:00100000:1.0:1767319594.950316:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982855296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319594.950325:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319594.950349:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319594.950355:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319594.950357:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982855296:12345-0@lo:38:kworker.0 Request processed in 38us (292us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319594.950361:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982855168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319594.950368:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 423 00000100:00080000:2.0:1767319594.950378:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319594.950383:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319594.950388:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982855296:0@lo:38:kworker.0 00000100:00080000:2.0:1767319594.950398:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319594.950401:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319594.950404:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982855680:0@lo:38:kworker.0 00010000:00010000:0.0:1767319595.141376:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319596.176856:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319596.940454:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319596.940499:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319596.940565:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633792 00000100:00100000:0.0:1767319596.940576:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633792 00000100:00100000:0.0:1767319596.940620:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915633920 00000100:00100000:0.0:1767319596.940623:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915633920 00000100:00100000:0.0:1767319596.940672:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1210 00000100:00100000:0.0:1767319596.940681:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915633920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319596.940751:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915633920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (251us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319596.940759:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1210 00000100:00100000:0.0:1767319596.940777:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1594 00000100:00100000:0.0:1767319596.940781:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915633792:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319596.940801:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319596.940853:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915633792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 70us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319596.940860:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1594 00000100:00080000:3.0:1767319596.940913:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319597.188641:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319597.508723:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982855808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319597.508810:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319597.508975:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982855936:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319597.508995:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855808 00000100:00100000:3.0:1767319597.509004:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855808 00000100:00100000:1.0:1767319597.509010:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319597.509061:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982855936 00000100:00100000:3.0:1767319597.509066:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982855936 00000100:00080000:1.0:1767319597.509069:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319597.509086:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319597.509086:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 820 00000100:00100000:3.0:1767319597.509111:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982855808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319597.509150:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319597.509189:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 821 00000100:00100000:3.0:1767319597.509199:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982855808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (387us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319597.509204:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982855936:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319597.509212:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 820 00002000:00000024:2.0:1767319597.509226:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319597.509294:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982855936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319597.509302:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 821 00000100:00100000:1.0:1767319597.509380:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982855936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319597.509569:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982855808:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319598.212521:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319599.236673:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319600.068459:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319600.068474:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319600.068710:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767319600.068733:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319600.068738:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319600.068743:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319600.068760:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4843 00000100:00080000:3.0:1767319600.068768:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319600.068777:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319600.068810:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580e00 x1853163982856192/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319600.068890:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982856192:0@lo:38:kworker.0 00000100:00100000:2.0:1767319600.068984:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319600.069804:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319600.069828:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982856320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319600.069885:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319600.069978:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856192 00000100:00100000:0.0:1767319600.069989:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856192 00000100:00100000:0.0:1767319600.070031:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856320 00000100:00100000:0.0:1767319600.070034:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856320 00000100:00100000:0.0:1767319600.070082:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211 00000100:00100000:0.0:1767319600.070093:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982856320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319600.070153:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982856320:12345-0@lo:400:kworker.0 Request processed in 61us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319600.070162:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211 00000100:00100000:0.0:1767319600.070181:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1595 00000100:00100000:0.0:1767319600.070186:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982856192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319600.070210:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319600.070242:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982856192:12345-0@lo:38:kworker.0 Request processed in 55us (1260us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319600.070249:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1595 00000100:00080000:0.0:1767319600.070303:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319600.070337:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580a80 x1853163982856320/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319616 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319600.070353:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982856320:0@lo:400:kworker.0 00000100:00080000:2.0:1767319600.071177:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319600.071186:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319600.071191:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982856192:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.073017:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982856064:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.073132:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319600.074164:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319600.074173:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319600.074178:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319600.074185:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4843 00000100:00080000:3.0:1767319600.074190:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319600.074199:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319600.074254:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583b80 x1853163982856448/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319600.074303:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319600.074306:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319600.074309:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319600.074312:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4843 00000100:00080000:3.0:1767319600.074315:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319600.074319:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319600.074335:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580e00 x1853163982856576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319600.074379:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319600.074383:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319600.074385:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319600.074389:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4843 00000100:00080000:3.0:1767319600.074392:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319600.074396:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319600.074412:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583100 x1853163982856960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319600.074453:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982856704:0@lo:400:kworker.0 00000100:00100000:0.0:1767319600.074634:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982856448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319600.074686:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.074764:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982856832:0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.074802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.074864:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856832 00000100:00100000:1.0:1767319600.074874:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856832 00000100:00100000:1.0:1767319600.074958:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 759 00000100:00100000:1.0:1767319600.074969:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982856832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.075036:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982856832:12345-0@lo:400:kworker.0 Request processed in 68us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319600.075044:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 759 00000100:00080000:1.0:1767319600.075069:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319600.075107:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982856832:0@lo:400:kworker.0 00000100:00100000:3.0:1767319600.075201:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.078413:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856704 00000100:00100000:1.0:1767319600.078435:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856704 00000100:00100000:1.0:1767319600.078563:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 760 00000100:00100000:1.0:1767319600.078576:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982856704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319600.078709:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982856704:12345-0@lo:400:kworker.0 Request processed in 135us (3506us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319600.078717:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 760 00000100:00080000:3.0:1767319600.079125:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319600.079179:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982856704:0@lo:400:kworker.0 00000100:00100000:0.0:1767319600.079459:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982856576:0@lo:38:kworker.0 00000100:00100000:0.0:1767319600.079526:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319600.079552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982856960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319600.079584:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319600.079639:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856576 00000100:00100000:1.0:1767319600.079646:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856576 00000100:00100000:1.0:1767319600.079680:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 424 00000100:00100000:1.0:1767319600.079688:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982856576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.079714:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.079764:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982856576:12345-0@lo:38:kworker.0 Request processed in 77us (237us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.079771:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 424 00000100:00100000:2.0:1767319600.080161:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856064 00000100:00100000:2.0:1767319600.080168:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856064 00000100:00100000:2.0:1767319600.080232:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856448 00000100:00100000:2.0:1767319600.080236:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856448 00000100:00100000:2.0:1767319600.080258:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212 00000100:00100000:2.0:1767319600.080266:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982856064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319600.080354:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982856064:12345-0@lo:400:kworker.0 Request processed in 90us (7223us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319600.080361:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212 00000100:00100000:2.0:1767319600.080378:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596 00000100:00100000:2.0:1767319600.080382:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982856448:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319600.080395:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319600.080424:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982856448:12345-0@lo:38:kworker.0 Request processed in 41us (5738us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319600.080431:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596 00000100:00080000:1.0:1767319600.080519:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319600.080561:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582680 x1853163982856064/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319616 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319600.080576:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982856064:0@lo:400:kworker.0 00000100:00080000:0.0:1767319600.080834:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319600.080841:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319600.080845:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982856448:0@lo:38:kworker.0 00000100:00080000:0.0:1767319600.080856:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319600.080860:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319600.080863:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982856576:0@lo:38:kworker.0 00000100:00100000:1.0:1767319600.081066:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982856960 00000100:00100000:1.0:1767319600.081073:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982856960 00000100:00100000:1.0:1767319600.081105:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1597 00000100:00100000:1.0:1767319600.081111:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982856960:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319600.081126:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319600.081176:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982856960:12345-0@lo:38:kworker.0 Request processed in 63us (1590us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319600.081182:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1597 00000100:00080000:0.0:1767319600.081244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319600.081249:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319600.081252:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982856960:0@lo:38:kworker.0 00010000:00010000:2.0:1767319600.262328:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319601.286324:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319602.063238:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319602.063344:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319602.063463:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634048 00000100:00100000:2.0:1767319602.063477:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634048 00000100:00100000:2.0:1767319602.063534:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634176 00000100:00100000:2.0:1767319602.063538:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634176 00000100:00080000:1.0:1767319602.063583:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319602.063606:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1213 00000100:00100000:2.0:1767319602.063620:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319602.063670:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1598 00000100:00100000:1.0:1767319602.063678:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634048:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319602.063727:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319602.063743:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 126us (397us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319602.063753:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1213 00000100:00100000:1.0:1767319602.063766:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634048:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 86us (544us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319602.063774:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1598 00010000:00010000:3.0:1767319602.308678:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319602.628661:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319602.628798:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319602.629033:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857088 00000100:00100000:2.0:1767319602.629045:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857088 00000100:00100000:2.0:1767319602.629123:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 822 00000100:00100000:2.0:1767319602.629134:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319602.629179:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319602.629241:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319602.629250:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 822 00000100:00100000:3.0:1767319602.629406:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857088:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319602.629493:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319602.630248:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319602.630264:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319602.630345:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319602.630419:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857216 00000100:00100000:0.0:1767319602.630427:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857216 00000100:00100000:0.0:1767319602.631841:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 823 00000100:00100000:0.0:1767319602.631860:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319602.631900:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319602.631996:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 144us (1650us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319602.632005:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 823 00000100:00100000:2.0:1767319602.632106:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857216:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319602.632840:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319603.332969:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319604.356527:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319605.188483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319605.188498:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319605.188567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319605.188572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319605.188577:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319605.188588:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4848 00000100:00080000:1.0:1767319605.188594:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319605.188602:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319605.188690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54a80 x1853163982857472/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319605.188722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319605.188726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319605.188728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319605.188733:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4848 00000100:00080000:1.0:1767319605.188736:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319605.188741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319605.188755:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163982857728/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319605.188764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319605.188767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319605.188769:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319605.188773:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4848 00000100:00080000:1.0:1767319605.188775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319605.188779:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319605.188793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56d80 x1853163982857856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319605.188836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319605.188840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319605.188843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319605.188847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4848 00000100:00080000:1.0:1767319605.188849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319605.188853:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319605.188861:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319605.188868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57480 x1853163982858240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319605.188954:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.189012:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982857344:0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.189045:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319605.189055:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982858112:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.189061:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319605.189092:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.189104:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319605.189113:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982857600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.189151:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982857472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319605.189154:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.189194:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857344 00000100:00100000:0.0:1767319605.189203:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857344 00000100:00100000:3.0:1767319605.189213:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319605.189247:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982857728:0@lo:38:kworker.0 00000100:00100000:3.0:1767319605.189292:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.189304:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857600 00000100:00100000:0.0:1767319605.189309:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857600 00000100:00100000:3.0:1767319605.189322:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982857856:0@lo:38:kworker.0 00000100:00100000:0.0:1767319605.189329:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857472 00000100:00100000:0.0:1767319605.189334:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857472 00000100:00100000:0.0:1767319605.189348:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214 00000100:00100000:3.0:1767319605.189353:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319605.189362:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982857344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.189379:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982858240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319605.189403:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319605.189433:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857728 00000100:00100000:0.0:1767319605.189434:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982857344:12345-0@lo:400:kworker.0 Request processed in 74us (374us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319605.189438:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857728 00000100:00100000:0.0:1767319605.189447:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214 00000100:00100000:3.0:1767319605.189457:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858240 00000100:00100000:3.0:1767319605.189460:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858240 00000100:00100000:0.0:1767319605.189472:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215 00000100:00100000:1.0:1767319605.189523:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857984 00000100:00100000:1.0:1767319605.189528:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857984 00000100:00100000:1.0:1767319605.189562:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858112 00000100:00100000:1.0:1767319605.189565:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858112 00000100:00100000:1.0:1767319605.189588:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 761 00000100:00100000:1.0:1767319605.189595:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319605.189651:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982857984:12345-0@lo:400:kworker.0 Request processed in 56us (697us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319605.189667:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 761 00000100:00100000:1.0:1767319605.189682:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 762 00000100:00100000:1.0:1767319605.189687:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982858112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319605.189713:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982858112:12345-0@lo:400:kworker.0 Request processed in 26us (623us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319605.189719:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 762 00000100:00100000:2.0:1767319605.189834:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982857856 00000100:00100000:2.0:1767319605.189839:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982857856 00000100:00100000:2.0:1767319605.189863:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 425 00000100:00100000:2.0:1767319605.189868:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982857856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319605.189889:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319605.189929:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982857856:12345-0@lo:38:kworker.0 Request processed in 59us (578us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319605.189936:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 425 00000100:00100000:0.0:1767319605.189993:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982857600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.190029:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982857600:12345-0@lo:400:kworker.0 Request processed in 549us (876us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319605.193854:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1599 00000100:00100000:3.0:1767319605.193875:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982857472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319605.193911:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319605.194029:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982857472:12345-0@lo:38:kworker.0 Request processed in 157us (4825us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319605.194037:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1599 00000100:00100000:3.0:1767319605.194079:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1600 00000100:00100000:3.0:1767319605.194083:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982857728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319605.194095:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319605.194127:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982857728:12345-0@lo:38:kworker.0 Request processed in 42us (4834us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319605.194133:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1600 00000100:00100000:3.0:1767319605.194149:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1601 00000100:00100000:3.0:1767319605.194153:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982858240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319605.194163:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319605.194187:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982858240:12345-0@lo:38:kworker.0 Request processed in 32us (4784us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319605.194193:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1601 00000100:00080000:1.0:1767319605.195933:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319605.196102:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319605.196106:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319605.196232:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57b80 x1853163982857600/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319621 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319605.196254:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982857600:0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.196368:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982858112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319605.196385:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982857984:0@lo:400:kworker.0 00000100:00080000:3.0:1767319605.196435:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319605.196440:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319605.196445:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982857472:0@lo:38:kworker.0 00000100:00080000:3.0:1767319605.196458:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319605.196462:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319605.196465:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982857728:0@lo:38:kworker.0 00000100:00080000:3.0:1767319605.196477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319605.196481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319605.196484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982857856:0@lo:38:kworker.0 00000100:00080000:3.0:1767319605.196494:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319605.196498:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319605.196501:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982858240:0@lo:38:kworker.0 00010000:00080000:1.0:1767319605.196533:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8e00 x1853163982857344/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319621 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319605.196545:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982857344:0@lo:400:kworker.0 00000100:00100000:0.0:1767319605.203025:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215 00010000:00010000:3.0:1767319605.382459:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319606.409143:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319607.182141:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319607.182363:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319607.182726:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634432 00000100:00100000:2.0:1767319607.182742:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634432 00000100:00100000:2.0:1767319607.182839:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634304 00000100:00100000:2.0:1767319607.182843:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634304 00000100:00100000:2.0:1767319607.182863:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1216 00000100:00100000:2.0:1767319607.182874:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319607.182988:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (865us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319607.182997:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1216 00000100:00100000:2.0:1767319607.183159:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1602 00000100:00100000:2.0:1767319607.183164:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915634304:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319607.183194:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319607.183231:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915634304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (869us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319607.183238:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1602 00000100:00080000:3.0:1767319607.184106:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319607.428506:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319607.749161:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982858368:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319607.749263:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319607.749592:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982858496:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319607.749675:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319607.749843:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858368 00000100:00100000:1.0:1767319607.749855:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858368 00000100:00100000:1.0:1767319607.749927:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858496 00000100:00100000:1.0:1767319607.749931:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858496 00000100:00100000:1.0:1767319607.749953:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 824 00000100:00100000:1.0:1767319607.749963:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982858368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319607.750003:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319607.750058:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982858368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (794us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319607.750066:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 824 00000100:00100000:1.0:1767319607.750088:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 825 00000100:00100000:1.0:1767319607.750095:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982858496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319607.750113:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319607.750145:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982858496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (469us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319607.750151:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 825 00000100:00100000:0.0:1767319607.750245:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982858368:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319607.750438:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319607.750443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319607.750471:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982858496:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767319608.454033:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319609.481915:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319610.308656:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319610.308672:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319610.308761:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319610.308766:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319610.308770:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319610.308781:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4853 00000100:00080000:3.0:1767319610.308789:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319610.308799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319610.308827:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580000 x1853163982858752/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319610.308889:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319610.308892:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319610.308894:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319610.308898:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4853 00000100:00080000:3.0:1767319610.308901:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319610.308906:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319610.308917:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163982859008/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319610.308925:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319610.308928:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319610.308930:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319610.308934:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4853 00000100:00080000:3.0:1767319610.308937:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319610.308940:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319610.308951:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903b80 x1853163982859136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319610.308988:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319610.308992:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319610.308994:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319610.308997:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4853 00000100:00080000:3.0:1767319610.309001:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319610.309004:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319610.309013:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901880 x1853163982859520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319610.309058:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982859392:0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.309148:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319610.309237:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319610.309244:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982859264:0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.309306:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319610.309348:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982858880:0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.309362:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859392 00000100:00100000:3.0:1767319610.309372:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859392 00000100:00100000:2.0:1767319610.309437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319610.309456:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859264 00000100:00100000:3.0:1767319610.309462:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859264 00000100:00100000:3.0:1767319610.309496:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 763 00000100:00100000:2.0:1767319610.309502:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319610.309509:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982859392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319610.309517:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982858624:0@lo:400:kworker.0 00000100:00100000:2.0:1767319610.309545:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319610.309619:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982859392:12345-0@lo:400:kworker.0 Request processed in 111us (467us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319610.309631:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 763 00000100:00100000:2.0:1767319610.309634:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858880 00000100:00100000:2.0:1767319610.309645:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858880 00000100:00100000:3.0:1767319610.309662:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 764 00000100:00100000:3.0:1767319610.309673:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463934a450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982859264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319610.309710:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217 00000100:00100000:3.0:1767319610.309718:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463934a450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982859264:12345-0@lo:400:kworker.0 Request processed in 46us (409us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319610.309727:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982858880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.309735:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 764 00000100:00100000:2.0:1767319610.309820:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982858880:12345-0@lo:400:kworker.0 Request processed in 100us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319610.309831:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217 00000100:00100000:2.0:1767319610.309855:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858624 00000100:00100000:2.0:1767319610.309859:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858624 00000100:00100000:2.0:1767319610.309902:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1218 00000100:00100000:2.0:1767319610.309911:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982858624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319610.309958:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982858624:12345-0@lo:400:kworker.0 Request processed in 49us (412us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319610.309967:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1218 00010000:00080000:2.0:1767319610.310042:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582680 x1853163982858624/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319626 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319610.310059:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982858624:0@lo:400:kworker.0 00010000:00080000:2.0:1767319610.310080:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900e00 x1853163982858880/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319626 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319610.310090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982858880:0@lo:400:kworker.0 00000100:00080000:3.0:1767319610.310108:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319610.310115:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319610.310118:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319610.310154:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982859264:0@lo:400:kworker.0 00000100:00100000:3.0:1767319610.310168:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982859392:0@lo:400:kworker.0 00000100:00100000:0.0:1767319610.310322:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982858752:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.310367:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.310406:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982859008:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.310434:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.310464:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982859136:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.310490:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.310512:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982859520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.310534:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319610.310583:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982858752 00000100:00100000:0.0:1767319610.310589:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982858752 00000100:00100000:0.0:1767319610.310614:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859008 00000100:00100000:0.0:1767319610.310617:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859008 00000100:00100000:0.0:1767319610.310624:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859520 00000100:00100000:0.0:1767319610.310626:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859520 00000100:00100000:0.0:1767319610.310637:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1603 00000100:00100000:0.0:1767319610.310889:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982858752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319610.310912:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319610.310969:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982858752:12345-0@lo:38:kworker.0 Request processed in 82us (600us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319610.314666:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859136 00000100:00100000:3.0:1767319610.314846:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859136 00000100:00100000:3.0:1767319610.315000:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 426 00000100:00100000:3.0:1767319610.315011:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982859136:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319610.315042:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319610.315691:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982859136:12345-0@lo:38:kworker.0 Request processed in 677us (5195us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319610.315703:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 426 00000100:00100000:1.0:1767319610.316634:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1604 00000100:00100000:1.0:1767319610.316646:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982859008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319610.316672:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319610.316738:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982859008:12345-0@lo:38:kworker.0 Request processed in 94us (6302us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319610.316746:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1604 00000100:00100000:1.0:1767319610.316774:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1605 00000100:00100000:1.0:1767319610.316778:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982859520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319610.316788:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319610.316815:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982859520:12345-0@lo:38:kworker.0 Request processed in 35us (6280us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319610.316821:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1605 00000100:00080000:1.0:1767319610.316930:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319610.316937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319610.316941:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982858752:0@lo:38:kworker.0 00000100:00080000:1.0:1767319610.316954:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319610.316957:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319610.316960:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982859008:0@lo:38:kworker.0 00000100:00080000:1.0:1767319610.316974:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319610.316978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319610.316981:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982859136:0@lo:38:kworker.0 00000100:00080000:1.0:1767319610.316990:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319610.316994:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319610.316997:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982859520:0@lo:38:kworker.0 00000100:00100000:0.0:1767319610.321633:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1603 00010000:00010000:0.0:1767319610.500464:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767319611.524743:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319612.300319:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319612.304432:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634560 00000100:00100000:2.0:1767319612.304449:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634560 00000100:00100000:3.0:1767319612.304519:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319612.304547:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1606 00000100:00100000:2.0:1767319612.304559:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634560:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319612.304588:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319612.304670:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634560:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 111us (4417us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319612.304679:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1606 00000100:00100000:2.0:1767319612.308230:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634688 00000100:00100000:2.0:1767319612.308246:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634688 00000100:00080000:3.0:1767319612.309032:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319612.310603:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1219 00000100:00100000:1.0:1767319612.310637:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634688:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319612.312772:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915634688:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 2138us (8247us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319612.312792:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1219 00000400:00000100:3.0:1767319612.313181:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915634816 offset 0 length 520: 4 00010000:00010000:2.0:1767319612.549023:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319612.868740:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982859648:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319612.868895:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319612.871464:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319612.872702:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859648 00000100:00100000:3.0:1767319612.872718:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859648 00000100:00100000:3.0:1767319612.872948:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 826 00000100:00100000:3.0:1767319612.872958:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982859648:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319612.872995:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767319612.873104:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319612.880230:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982859648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 7260us (11322us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319612.880248:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 826 00000100:00100000:1.0:1767319612.882648:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982859776:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319612.884208:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319612.884488:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982859776 00000100:00100000:3.0:1767319612.884503:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982859776 00000100:00100000:1.0:1767319612.884533:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982859648:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319612.884628:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827 00000100:00100000:3.0:1767319612.884639:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934602e4d450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982859776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319612.884673:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319612.884933:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934602e4d450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982859776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 296us (733us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319612.884941:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827 00000100:00080000:0.0:1767319612.885876:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319612.890934:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982859776:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319613.572435:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319614.404754:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319589/real 1767319589] req@ffff934604581180 x1853163982854400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319614 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319614.404791:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604581180 x1853163982854400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319614 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319614.404815:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319614.405081:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319614.405086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982854400:0@lo:250:kworker.0 00000100:00080000:2.0:1767319614.408612:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319614.408624:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767319614.408628:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319614.408635:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4838 00000100:00080000:2.0:1767319614.408640:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319614.408647:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319614.408692:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163982859904/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319614.408786:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982859904:0@lo:250:kworker.0 00000400:00000100:3.0:1767319614.408843:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982859904 offset 0 length 520: 4 00000100:00080000:2.0:1767319614.408944:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319614.408948:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319614.408951:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319614.408955:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4858 00000100:00080000:2.0:1767319614.408958:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319614.408962:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319614.408981:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163982860160/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319614.409042:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319614.409056:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982860032:0@lo:400:kworker.0 00000100:00100000:0.0:1767319614.409096:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319614.410654:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860032 00000100:00100000:1.0:1767319614.410667:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860032 00000100:00100000:1.0:1767319614.410748:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220 00000100:00100000:1.0:1767319614.410759:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982860032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319614.410837:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982860032:12345-0@lo:400:kworker.0 Request processed in 81us (1740us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319614.410845:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220 00000100:00080000:0.0:1767319614.410944:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319614.410981:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc380 x1853163982860032/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319630 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319614.410996:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982860032:0@lo:400:kworker.0 00000100:00100000:3.0:1767319614.411720:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982860160:0@lo:38:kworker.0 00000100:00100000:3.0:1767319614.411765:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319614.418036:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860160 00000100:00100000:1.0:1767319614.418053:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860160 00000100:00100000:1.0:1767319614.418111:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1607 00000100:00100000:1.0:1767319614.418120:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982860160:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319614.418141:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319614.418239:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982860160:12345-0@lo:38:kworker.0 Request processed in 120us (6473us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319614.418246:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1607 00000100:00080000:0.0:1767319614.418390:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319614.418397:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319614.418401:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982860160:0@lo:38:kworker.0 00000100:00080000:2.0:1767319614.418784:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319614.418789:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319614.418792:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319614.418796:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4858 00000100:00080000:2.0:1767319614.418800:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319614.418986:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319614.419003:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982860288:0@lo:400:kworker.0 00000100:00100000:3.0:1767319614.419048:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319614.419108:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319614.419254:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdc00 x1853163982860416/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319614.419293:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319614.419297:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319614.419299:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319614.419303:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4858 00000100:00080000:2.0:1767319614.419305:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319614.419309:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319614.419321:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163982860544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319614.419348:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319614.419351:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319614.419353:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319614.419356:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4858 00000100:00080000:2.0:1767319614.419358:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319614.419361:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319614.419369:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163982860928/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319614.419390:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982860800:0@lo:400:kworker.0 00000100:00100000:2.0:1767319614.419420:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319614.419450:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319614.419456:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982860672:0@lo:400:kworker.0 00000100:00100000:2.0:1767319614.419474:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319614.419564:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860800 00000100:00100000:1.0:1767319614.419570:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860800 00000100:00100000:1.0:1767319614.419624:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860672 00000100:00100000:1.0:1767319614.419627:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860672 00000100:00100000:1.0:1767319614.419646:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 765 00000100:00100000:1.0:1767319614.419658:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982860800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319614.419708:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982860800:12345-0@lo:400:kworker.0 Request processed in 55us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319614.419714:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 765 00000100:00100000:1.0:1767319614.419729:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 766 00000100:00100000:1.0:1767319614.419735:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982860672:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319614.419761:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982860672:12345-0@lo:400:kworker.0 Request processed in 27us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319614.419766:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 766 00000100:00100000:1.0:1767319614.419805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982860672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319614.419813:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982860800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319614.420051:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982860416:0@lo:38:kworker.0 00000100:00100000:3.0:1767319614.420080:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319614.420113:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319614.420118:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319614.427739:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982860544:0@lo:38:kworker.0 00000100:00100000:3.0:1767319614.427795:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319614.427811:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982860928:0@lo:38:kworker.0 00000100:00100000:3.0:1767319614.427829:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319614.427920:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860544 00000100:00100000:2.0:1767319614.427930:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860544 00000100:00100000:2.0:1767319614.427966:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 427 00000100:00100000:2.0:1767319614.427973:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982860544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319614.427993:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319614.428030:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982860544:12345-0@lo:38:kworker.0 Request processed in 57us (234us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319614.428037:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 427 00000100:00100000:0.0:1767319614.430065:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860288 00000100:00100000:0.0:1767319614.430077:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860288 00000100:00100000:0.0:1767319614.430146:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860416 00000100:00100000:0.0:1767319614.430150:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860416 00000100:00100000:0.0:1767319614.430178:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221 00000100:00100000:0.0:1767319614.430187:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982860288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319614.430267:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982860288:12345-0@lo:400:kworker.0 Request processed in 80us (11217us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319614.430292:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221 00000100:00100000:0.0:1767319614.430309:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1608 00000100:00100000:0.0:1767319614.430313:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982860416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319614.430329:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319614.430356:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982860416:12345-0@lo:38:kworker.0 Request processed in 42us (10276us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319614.430361:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1608 00010000:00080000:0.0:1767319614.430410:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff800 x1853163982860288/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319630 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319614.430423:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982860288:0@lo:400:kworker.0 00000100:00080000:2.0:1767319614.430557:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319614.430978:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319614.430984:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319614.430986:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982860416:0@lo:38:kworker.0 00000100:00080000:3.0:1767319614.430994:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319614.430996:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319614.430998:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982860544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319614.431053:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982860928 00000100:00100000:1.0:1767319614.431058:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982860928 00000100:00100000:0.0:1767319614.432962:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1609 00000100:00100000:0.0:1767319614.432971:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982860928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319614.432992:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319614.433074:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982860928:12345-0@lo:38:kworker.0 Request processed in 101us (5242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319614.433081:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1609 00000100:00080000:3.0:1767319614.433137:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319614.433143:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319614.433147:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982860928:0@lo:38:kworker.0 00010000:00010000:3.0:1767319614.597200:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319615.621209:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319616.644393:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319617.419506:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319617.419789:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635072 00000100:00100000:1.0:1767319617.419800:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635072 00000100:00100000:1.0:1767319617.419890:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1222 00000100:00100000:1.0:1767319617.419901:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915635072:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319617.419993:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319617.420743:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915635072:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 840us (1253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319617.420753:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1222 00000100:00100000:3.0:1767319617.421288:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319617.421364:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915634944 00000100:00100000:1.0:1767319617.421372:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915634944 00000100:00100000:1.0:1767319617.421450:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1610 00000100:00100000:1.0:1767319617.421457:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634944:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319617.421479:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319617.421546:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915634944:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319617.421553:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1610 00010000:00010000:3.0:1767319617.668539:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319617.989096:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982861056:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319617.989145:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982861184:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319617.989172:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319617.989207:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319617.989307:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861056 00000100:00100000:2.0:1767319617.989315:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861056 00000100:00100000:2.0:1767319617.989379:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861184 00000100:00100000:2.0:1767319617.989383:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861184 00000100:00100000:2.0:1767319617.989405:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 828 00000100:00080000:3.0:1767319617.989408:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319617.989415:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982861056:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319617.989415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767319617.989460:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319617.989525:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982861056:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (345us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319617.989534:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 828 00000100:00100000:2.0:1767319617.989552:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 829 00000100:00100000:2.0:1767319617.989561:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982861184:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319617.989561:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982861056:0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319617.989583:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319617.989627:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982861184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (420us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319617.989634:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 829 00000100:00100000:3.0:1767319617.989686:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982861184:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319618.694009:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319619.525342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319619.525351:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319619.525424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319619.525428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319619.525431:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319619.525439:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4862 00000100:00080000:0.0:1767319619.525445:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319619.525452:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319619.525474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965500 x1853163982861440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319619.525502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319619.525505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319619.525507:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319619.525512:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4862 00000100:00080000:0.0:1767319619.525514:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319619.525518:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319619.525527:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982861312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319619.525534:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962d80 x1853163982861696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319619.525543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319619.525546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319619.525547:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319619.525550:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4862 00000100:00080000:0.0:1767319619.525553:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319619.525556:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319619.525569:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960e00 x1853163982861824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319619.525601:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319619.525611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319619.525614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319619.525616:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319619.525620:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4862 00000100:00080000:0.0:1767319619.525623:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319619.525628:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319619.525639:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91880 x1853163982862208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319619.525645:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319619.525656:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982861568:0@lo:400:kworker.0 00000100:00100000:0.0:1767319619.525668:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982861952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319619.525684:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319619.525712:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319619.525724:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861312 00000100:00100000:1.0:1767319619.525732:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861312 00000100:00100000:0.0:1767319619.525751:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319619.525758:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982862080:0@lo:400:kworker.0 00000100:00100000:0.0:1767319619.525781:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319619.525785:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861568 00000100:00100000:1.0:1767319619.525790:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861568 00000100:00100000:1.0:1767319619.525821:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223 00000100:00100000:1.0:1767319619.525831:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982861312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319619.525848:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861952 00000100:00100000:0.0:1767319619.525854:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861952 00000100:00100000:1.0:1767319619.525882:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982861312:12345-0@lo:400:kworker.0 Request processed in 52us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319619.525892:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862080 00000100:00100000:1.0:1767319619.525892:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223 00000100:00100000:0.0:1767319619.525897:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862080 00000100:00100000:1.0:1767319619.525915:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224 00000100:00100000:1.0:1767319619.525924:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982861568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319619.525929:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 767 00000100:00100000:0.0:1767319619.525941:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982861952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319619.525957:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982861568:12345-0@lo:400:kworker.0 Request processed in 34us (273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319619.525964:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224 00000100:00080000:1.0:1767319619.525982:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319619.525986:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319619.525989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319619.525993:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982861952:12345-0@lo:400:kworker.0 Request processed in 52us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319619.526002:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 767 00000100:00100000:3.0:1767319619.526090:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982861440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319619.526122:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319619.526143:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982861696:0@lo:38:kworker.0 00000100:00100000:3.0:1767319619.526163:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319619.526180:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982861824:0@lo:38:kworker.0 00000100:00100000:3.0:1767319619.526207:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319619.526224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982862208:0@lo:38:kworker.0 00000100:00100000:3.0:1767319619.526245:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319619.526268:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861824 00000100:00100000:3.0:1767319619.526289:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861824 00000100:00100000:3.0:1767319619.526311:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 428 00000100:00100000:3.0:1767319619.526315:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638756850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982861824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319619.526332:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319619.526361:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638756850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982861824:12345-0@lo:38:kworker.0 Request processed in 44us (154us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319619.526367:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 428 00000100:00080000:3.0:1767319619.526535:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319619.526540:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319619.526543:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982861824:0@lo:38:kworker.0 00000100:00100000:3.0:1767319619.526570:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861440 00000100:00100000:3.0:1767319619.526574:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861440 00000100:00100000:3.0:1767319619.526585:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982861696 00000100:00100000:3.0:1767319619.526588:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982861696 00000100:00100000:3.0:1767319619.526594:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862208 00000100:00100000:3.0:1767319619.526596:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862208 00000100:00100000:3.0:1767319619.526604:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1611 00000100:00100000:3.0:1767319619.526608:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982861440:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319619.526617:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319619.526643:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982861440:12345-0@lo:38:kworker.0 Request processed in 34us (522us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319619.526649:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1611 00000100:00100000:3.0:1767319619.526659:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1612 00000100:00100000:3.0:1767319619.526663:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982861696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319619.526672:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319619.526692:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982861696:12345-0@lo:38:kworker.0 Request processed in 28us (528us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319619.526698:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1612 00000100:00100000:3.0:1767319619.526708:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1613 00000100:00100000:3.0:1767319619.526711:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982862208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319619.526720:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319619.526740:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982862208:12345-0@lo:38:kworker.0 Request processed in 27us (495us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319619.526746:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1613 00000100:00080000:3.0:1767319619.526767:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319619.526771:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319619.526774:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982861440:0@lo:38:kworker.0 00000100:00080000:3.0:1767319619.526782:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319619.526786:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319619.526788:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982861696:0@lo:38:kworker.0 00000100:00080000:3.0:1767319619.526795:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319619.526798:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319619.526800:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982862208:0@lo:38:kworker.0 00000100:00100000:1.0:1767319619.527519:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 768 00000100:00100000:1.0:1767319619.527533:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982862080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319619.527576:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982862080:12345-0@lo:400:kworker.0 Request processed in 49us (1794us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319619.527582:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 768 00000100:00100000:1.0:1767319619.527634:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982862080:0@lo:400:kworker.0 00000100:00100000:1.0:1767319619.527646:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982861952:0@lo:400:kworker.0 00010000:00080000:1.0:1767319619.527691:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f966d80 x1853163982861568/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319635 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319619.527701:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982861568:0@lo:400:kworker.0 00010000:00080000:1.0:1767319619.527712:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f967800 x1853163982861312/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319635 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319619.527719:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982861312:0@lo:400:kworker.0 00010000:00010000:0.0:1767319619.716456:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319620.741592:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319621.764621:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319622.549253:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319622.549358:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319622.549556:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635328 00000100:00100000:2.0:1767319622.549567:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635328 00000100:00100000:2.0:1767319622.549639:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635200 00000100:00100000:2.0:1767319622.549643:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635200 00000100:00100000:2.0:1767319622.549661:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1225 00000100:00100000:2.0:1767319622.549670:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915635328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319622.549746:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915635328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (504us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319622.549753:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1225 00000100:00100000:2.0:1767319622.549771:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1614 00000100:00100000:2.0:1767319622.549776:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915635200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319622.549795:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319622.549821:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915635200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319622.549827:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1614 00000100:00080000:3.0:1767319622.550034:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:0.0:1767319622.789758:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319623.108649:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982862336:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319623.108932:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319623.109013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982862464:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319623.109045:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319623.109096:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862336 00000100:00100000:2.0:1767319623.109103:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862336 00000100:00100000:2.0:1767319623.109175:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862464 00000100:00100000:2.0:1767319623.109180:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862464 00000100:00100000:2.0:1767319623.109201:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830 00000100:00100000:2.0:1767319623.109209:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982862336:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319623.109242:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319623.109450:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767319623.109460:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982862464:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319623.109478:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767319623.109507:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319623.109514:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319623.109521:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982862464:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (474us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319623.109535:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767319623.109586:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982862464:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319623.110073:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982862336:12345-0@lo:13:osp-pre-0-1.0 Request processed in 862us (1139us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319623.110082:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830 00000100:00100000:1.0:1767319623.110303:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982862336:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319623.812657:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319624.644475:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319624.644487:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319624.644579:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767319624.645608:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319624.645616:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319624.645620:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319624.645629:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4868 00000100:00080000:2.0:1767319624.645633:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319624.645639:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319624.645672:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd500 x1853163982862720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319624.645780:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982862720:0@lo:38:kworker.0 00000100:00100000:0.0:1767319624.645847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319624.646354:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862720 00000100:00100000:1.0:1767319624.646363:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862720 00000100:00100000:1.0:1767319624.646412:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1615 00000100:00100000:1.0:1767319624.646419:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982862720:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319624.646440:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319624.646493:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982862720:12345-0@lo:38:kworker.0 Request processed in 72us (645us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319624.646500:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1615 00000100:00080000:1.0:1767319624.646547:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319624.646552:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319624.646555:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982862720:0@lo:38:kworker.0 00000100:00100000:3.0:1767319624.648595:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982862848:0@lo:400:kworker.0 00000100:00100000:3.0:1767319624.648670:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319624.648702:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982862592:0@lo:400:kworker.0 00000100:00100000:3.0:1767319624.648723:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319624.648884:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862848 00000100:00100000:0.0:1767319624.648892:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862848 00000100:00100000:0.0:1767319624.648948:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862592 00000100:00100000:0.0:1767319624.648951:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862592 00000100:00100000:0.0:1767319624.648983:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1226 00000100:00100000:0.0:1767319624.648992:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982862848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319624.649041:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982862848:12345-0@lo:400:kworker.0 Request processed in 51us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319624.649048:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1226 00000100:00100000:0.0:1767319624.649064:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227 00000100:00100000:0.0:1767319624.649069:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982862592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319624.649092:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982862592:12345-0@lo:400:kworker.0 Request processed in 24us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319624.649097:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227 00000100:00080000:0.0:1767319624.649147:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319624.649537:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fed80 x1853163982862592/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319640 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319624.649551:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982862592:0@lo:400:kworker.0 00010000:00080000:3.0:1767319624.649576:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ffb80 x1853163982862848/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319640 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319624.649584:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982862848:0@lo:400:kworker.0 00000100:00080000:2.0:1767319624.649698:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319624.649704:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319624.649708:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319624.649713:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4868 00000100:00080000:2.0:1767319624.649717:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319624.649721:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319624.649884:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163982862976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319624.649910:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319624.649914:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319624.649916:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319624.649919:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4868 00000100:00080000:2.0:1767319624.649922:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319624.649926:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319624.649941:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163982863104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319624.649998:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982863232:0@lo:400:kworker.0 00000100:00100000:0.0:1767319624.650034:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319624.650143:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863232 00000100:00100000:3.0:1767319624.650149:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863232 00000100:00100000:3.0:1767319624.650182:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 769 00000100:00100000:3.0:1767319624.650189:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982863232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319624.650258:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982862976:0@lo:38:kworker.0 00000100:00100000:1.0:1767319624.650316:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319624.650336:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982863104:0@lo:38:kworker.0 00000100:00100000:1.0:1767319624.650400:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319624.650439:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863104 00000100:00100000:1.0:1767319624.650443:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863104 00000100:00100000:1.0:1767319624.650464:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 429 00000100:00100000:1.0:1767319624.650470:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982863104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319624.650488:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319624.650522:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982863104:12345-0@lo:38:kworker.0 Request processed in 51us (122us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319624.650527:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 429 00000100:00100000:1.0:1767319624.650549:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982862976 00000100:00100000:1.0:1767319624.650552:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982862976 00000100:00100000:1.0:1767319624.650571:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1616 00000100:00100000:1.0:1767319624.650576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982862976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319624.650585:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319624.650609:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982862976:12345-0@lo:38:kworker.0 Request processed in 31us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319624.650614:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1616 00000100:00080000:1.0:1767319624.650641:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319624.650645:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319624.650648:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982862976:0@lo:38:kworker.0 00000100:00080000:1.0:1767319624.650656:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319624.650659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319624.650662:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982863104:0@lo:38:kworker.0 00000100:00100000:1.0:1767319624.650691:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982863360:0@lo:400:kworker.0 00000100:00100000:1.0:1767319624.650714:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319624.650759:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863360 00000100:00100000:1.0:1767319624.650763:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863360 00000100:00100000:1.0:1767319624.650789:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 770 00000100:00100000:1.0:1767319624.650796:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982863360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319624.650829:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982863360:12345-0@lo:400:kworker.0 Request processed in 33us (114us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319624.650835:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 770 00000100:00100000:1.0:1767319624.650864:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982863360:0@lo:400:kworker.0 00000100:00080000:2.0:1767319624.651168:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319624.651172:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319624.651175:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319624.651179:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4868 00000100:00080000:2.0:1767319624.651182:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319624.651186:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319624.651206:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982863488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319624.651327:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982863488:0@lo:38:kworker.0 00000100:00100000:1.0:1767319624.651354:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319624.651432:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863488 00000100:00100000:2.0:1767319624.651438:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863488 00000100:00100000:2.0:1767319624.651464:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1617 00000100:00100000:2.0:1767319624.651468:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982863488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319624.651479:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319624.651510:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982863488:12345-0@lo:38:kworker.0 Request processed in 41us (155us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319624.651516:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1617 00000100:00080000:1.0:1767319624.651582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319624.651586:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319624.651589:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982863488:0@lo:38:kworker.0 00000100:00100000:3.0:1767319624.651983:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982863232:12345-0@lo:400:kworker.0 Request processed in 1793us (1945us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319624.651992:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 769 00000100:00100000:0.0:1767319624.652213:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982863232:0@lo:400:kworker.0 00010000:00010000:2.0:1767319624.836504:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319625.861790:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319626.884556:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319627.667885:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319627.667946:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319627.673802:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635584 00000100:00100000:2.0:1767319627.673819:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635584 00000100:00100000:2.0:1767319627.673918:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635456 00000100:00100000:2.0:1767319627.673921:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635456 00000100:00100000:2.0:1767319627.673950:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1228 00000100:00100000:2.0:1767319627.673961:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915635584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319627.674047:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915635584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (6173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319627.674055:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1228 00000100:00100000:2.0:1767319627.674075:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1618 00000100:00100000:2.0:1767319627.674080:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915635456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319627.674099:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319627.674125:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915635456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (6179us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319627.674132:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1618 00000100:00080000:0.0:1767319627.674904:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319627.911703:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319628.228917:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319628.228938:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982863616:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319628.229007:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319628.229152:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863616 00000100:00100000:2.0:1767319628.229160:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863616 00000100:00100000:2.0:1767319628.229227:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 832 00000100:00100000:2.0:1767319628.229247:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982863616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319628.229307:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319628.229354:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982863616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319628.229361:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 832 00000100:00080000:3.0:1767319628.230574:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319628.230633:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982863616:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319628.232458:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319628.232481:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982863744:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319628.232548:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319628.232662:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863744 00000100:00100000:0.0:1767319628.232670:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863744 00000100:00100000:0.0:1767319628.232744:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 833 00000100:00100000:0.0:1767319628.232755:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982863744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319628.232784:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319628.232831:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982863744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319628.232849:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 833 00000100:00080000:1.0:1767319628.232912:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319628.232949:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982863744:0@lo:13:osp-pre-0-1.0 00010000:00010000:0.0:1767319628.932459:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319629.764381:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319629.764391:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319629.764457:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319629.764461:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319629.764465:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319629.764473:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4873 00000100:00080000:2.0:1767319629.764479:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319629.764485:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319629.764512:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fca80 x1853163982864000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319629.764513:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319629.764528:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982863872:0@lo:400:kworker.0 00000100:00080000:2.0:1767319629.764548:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319629.764550:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319629.764552:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319629.764555:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4873 00000100:00080000:2.0:1767319629.764557:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319629.764560:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319629.764570:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982864256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319629.764578:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319629.764581:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319629.764582:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319629.764585:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4873 00000100:00080000:2.0:1767319629.764587:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319629.764590:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319629.764593:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319629.764599:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdc00 x1853163982864384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319629.764626:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319629.764629:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319629.764631:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319629.764634:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4873 00000100:00080000:2.0:1767319629.764637:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319629.764641:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319629.764649:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ffb80 x1853163982864768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319629.764657:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982864128:0@lo:400:kworker.0 00000100:00100000:1.0:1767319629.764678:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319629.764686:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982863872 00000100:00100000:2.0:1767319629.764692:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982863872 00000100:00100000:1.0:1767319629.764700:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982864512:0@lo:400:kworker.0 00000100:00100000:2.0:1767319629.764743:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864128 00000100:00100000:2.0:1767319629.764747:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864128 00000100:00100000:2.0:1767319629.764770:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229 00000100:00100000:2.0:1767319629.764779:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982863872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319629.764835:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982863872:12345-0@lo:400:kworker.0 Request processed in 56us (241us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319629.764843:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229 00000100:00100000:3.0:1767319629.764848:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982864640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319629.764860:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1230 00000100:00100000:2.0:1767319629.764865:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982864128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319629.764878:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319629.764888:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982864128:12345-0@lo:400:kworker.0 Request processed in 24us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319629.764894:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1230 00000100:00080000:2.0:1767319629.764908:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319629.764913:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864640 00000100:00100000:3.0:1767319629.764916:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864640 00010000:00080000:2.0:1767319629.764938:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe680 x1853163982864128/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319645 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319629.764942:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 771 00000100:00100000:3.0:1767319629.764947:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982864640:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319629.764952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982864128:0@lo:400:kworker.0 00010000:00080000:2.0:1767319629.764963:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fce00 x1853163982863872/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319645 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319629.764971:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982863872:0@lo:400:kworker.0 00000100:00100000:3.0:1767319629.764976:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982864640:12345-0@lo:400:kworker.0 Request processed in 29us (98us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319629.764981:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 771 00000100:00100000:2.0:1767319629.765003:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982864000:0@lo:38:kworker.0 00000100:00100000:3.0:1767319629.765007:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982864640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319629.765025:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319629.765045:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982864256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319629.765046:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864000 00000100:00100000:3.0:1767319629.765049:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864000 00000100:00100000:2.0:1767319629.765064:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319629.765065:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1619 00000100:00100000:3.0:1767319629.765070:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864000:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319629.765081:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982864384:0@lo:38:kworker.0 00010000:00020000:3.0:1767319629.765086:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319629.765101:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319629.765109:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864000:12345-0@lo:38:kworker.0 Request processed in 39us (84us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319629.765115:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1619 00000100:00100000:2.0:1767319629.765119:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982864768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319629.765123:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864256 00000100:00100000:3.0:1767319629.765126:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864256 00000100:00100000:3.0:1767319629.765139:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620 00000100:00100000:2.0:1767319629.765140:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319629.765143:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319629.765152:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319629.765157:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319629.765162:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319629.765165:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982864000:0@lo:38:kworker.0 00000100:00080000:2.0:1767319629.765173:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319629.765173:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864256:12345-0@lo:38:kworker.0 Request processed in 28us (108us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319629.765176:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319629.765178:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982864256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319629.765179:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620 00000100:00100000:3.0:1767319629.765186:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864768 00000100:00100000:3.0:1767319629.765189:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864768 00000100:00100000:3.0:1767319629.765221:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1621 00000100:00100000:3.0:1767319629.765225:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319629.765235:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319629.765303:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982864768:12345-0@lo:38:kworker.0 Request processed in 76us (161us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319629.765309:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1621 00000100:00100000:3.0:1767319629.765333:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864384 00000100:00100000:3.0:1767319629.765337:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864384 00000100:00100000:3.0:1767319629.765357:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 430 00000100:00100000:3.0:1767319629.765360:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982864384:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319629.765369:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319629.765405:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982864384:12345-0@lo:38:kworker.0 Request processed in 43us (303us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319629.765411:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 430 00000100:00100000:1.0:1767319629.765488:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319629.765570:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319629.765575:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319629.765578:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982864384:0@lo:38:kworker.0 00000100:00080000:2.0:1767319629.765586:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319629.765588:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319629.765591:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982864768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319629.765644:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864512 00000100:00100000:3.0:1767319629.765648:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864512 00000100:00100000:3.0:1767319629.765680:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 772 00000100:00100000:3.0:1767319629.765687:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982864512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319629.765729:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982864512:12345-0@lo:400:kworker.0 Request processed in 43us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319629.765735:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 772 00000100:00100000:1.0:1767319629.765968:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982864512:0@lo:400:kworker.0 00010000:00010000:1.0:1767319629.957919:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319630.980542:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319632.008539:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319632.793118:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319632.793190:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319632.793312:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635840 00000100:00100000:0.0:1767319632.793324:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635840 00000100:00100000:0.0:1767319632.793405:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915635712 00000100:00100000:0.0:1767319632.793409:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915635712 00000100:00100000:0.0:1767319632.793429:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1231 00000100:00100000:0.0:1767319632.793440:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915635840:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319632.793540:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915635840:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (434us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319632.793548:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1231 00000100:00100000:0.0:1767319632.793566:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1622 00000100:00100000:0.0:1767319632.793571:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915635712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319632.793591:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319632.793867:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915635712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 293us (676us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319632.793875:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1622 00000100:00080000:2.0:1767319632.794903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319633.030664:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319633.348524:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319633.348546:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982864896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319633.348619:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319633.348751:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982864896 00000100:00100000:2.0:1767319633.348761:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982864896 00000100:00100000:3.0:1767319633.348808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865024:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319633.348833:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319633.348870:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865024 00000100:00100000:3.0:1767319633.348874:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865024 00000100:00100000:2.0:1767319633.348947:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 834 00000100:00100000:2.0:1767319633.348957:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982864896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319633.348986:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319633.349044:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982864896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (425us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319633.349052:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 834 00000100:00100000:2.0:1767319633.349098:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982864896:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319633.353188:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319633.353199:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319633.357024:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 835 00000100:00100000:3.0:1767319633.357039:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319633.357069:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319633.357166:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 133us (8331us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319633.357172:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 835 00000100:00100000:1.0:1767319633.357249:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865024:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319634.052574:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319634.884395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319634.884407:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319634.884483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319634.884487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319634.884491:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319634.884511:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4878 00000100:00080000:0.0:1767319634.884516:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319634.884523:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319634.884547:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163982865280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319634.884591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319634.884594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319634.884596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319634.884600:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4878 00000100:00080000:0.0:1767319634.884602:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319634.884606:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319634.884620:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912a00 x1853163982865536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319634.884629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319634.884633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767319634.884636:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982865280:0@lo:38:kworker.0 00000100:00080000:0.0:1767319634.884637:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319634.884645:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4878 00000100:00080000:0.0:1767319634.884648:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319634.884651:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319634.884666:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913b80 x1853163982865664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319634.884704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319634.884708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319634.884711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319634.884716:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319634.884730:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4878 00000100:00080000:0.0:1767319634.884734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319634.884738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319634.884751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163982866048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319634.884779:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319634.884794:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982865536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319634.884818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319634.884831:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319634.884859:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319634.884863:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982865664:0@lo:38:kworker.0 00000100:00100000:0.0:1767319634.884870:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865792:0@lo:400:kworker.0 00000100:00100000:3.0:1767319634.884898:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319634.884922:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319634.884926:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982866048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319634.884957:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319634.884993:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865664 00000100:00100000:0.0:1767319634.885002:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865664 00000100:00100000:3.0:1767319634.885006:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865920 00000100:00100000:3.0:1767319634.885012:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865920 00000100:00100000:0.0:1767319634.885041:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 431 00000100:00100000:0.0:1767319634.885049:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982865664:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319634.885065:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865792 00010000:00020000:0.0:1767319634.885070:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319634.885071:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865792 00000100:00100000:3.0:1767319634.885098:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 773 00000100:00100000:3.0:1767319634.885107:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865920:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319634.885131:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982865664:12345-0@lo:38:kworker.0 Request processed in 79us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319634.885139:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 431 00000100:00100000:0.0:1767319634.885162:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865280 00000100:00100000:0.0:1767319634.885168:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865280 00000100:00100000:3.0:1767319634.885173:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865920:12345-0@lo:400:kworker.0 Request processed in 66us (354us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319634.885182:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 773 00000100:00100000:0.0:1767319634.885193:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865536 00000100:00100000:0.0:1767319634.885198:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865536 00000100:00100000:3.0:1767319634.885202:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 774 00000100:00100000:3.0:1767319634.885211:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865792:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319634.885215:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866048 00000100:00100000:0.0:1767319634.885219:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866048 00000100:00100000:0.0:1767319634.885232:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1623 00000100:00100000:0.0:1767319634.885238:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982865280:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319634.885246:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982865792:12345-0@lo:400:kworker.0 Request processed in 35us (325us total) trans 0 rc 0/0 00010000:00020000:0.0:1767319634.885252:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319634.885257:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 774 00000100:00100000:0.0:1767319634.885310:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982865280:12345-0@lo:38:kworker.0 Request processed in 70us (596us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319634.885317:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1623 00000100:00100000:3.0:1767319634.885327:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1624 00000100:00100000:0.0:1767319634.885336:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1625 00000100:00100000:3.0:1767319634.885337:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982865536:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319634.885347:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982866048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319634.885353:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767319634.885360:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319634.885393:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982865536:12345-0@lo:38:kworker.0 Request processed in 57us (562us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319634.885400:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982866048:12345-0@lo:38:kworker.0 Request processed in 53us (442us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319634.885403:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1624 00000100:00100000:0.0:1767319634.885413:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1625 00000100:00100000:0.0:1767319634.885472:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865792:0@lo:400:kworker.0 00000100:00080000:3.0:1767319634.885478:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319634.885485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319634.885487:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982865920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319634.885492:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982865536:0@lo:38:kworker.0 00000100:00080000:3.0:1767319634.885511:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319634.885515:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319634.885519:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982865664:0@lo:38:kworker.0 00000100:00080000:3.0:1767319634.885529:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319634.885532:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319634.885535:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982866048:0@lo:38:kworker.0 00000100:00080000:3.0:1767319634.885543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319634.885546:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319634.885549:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982865280:0@lo:38:kworker.0 00000100:00100000:2.0:1767319634.885949:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982865408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319634.885986:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982865152:0@lo:400:kworker.0 00000100:00100000:2.0:1767319634.886001:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319634.886030:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319634.886125:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865408 00000100:00100000:0.0:1767319634.886131:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865408 00000100:00100000:0.0:1767319634.886181:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982865152 00000100:00100000:0.0:1767319634.886184:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982865152 00000100:00080000:1.0:1767319634.886232:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319634.888285:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232 00000100:00100000:0.0:1767319634.888309:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982865408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319634.888400:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982865408:12345-0@lo:400:kworker.0 Request processed in 103us (2407us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319634.888410:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232 00000100:00100000:2.0:1767319634.888606:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233 00000100:00100000:2.0:1767319634.888619:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982865152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319634.888672:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982865152:12345-0@lo:400:kworker.0 Request processed in 56us (2641us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319634.888679:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233 00010000:00080000:2.0:1767319634.888726:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912d80 x1853163982865408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319650 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319634.888742:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982865408:0@lo:400:kworker.0 00010000:00080000:2.0:1767319634.888800:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911180 x1853163982865152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319650 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319634.888811:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982865152:0@lo:400:kworker.0 00010000:00010000:0.0:1767319635.076428:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319636.102624:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319636.883720:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319636.883761:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915635968 offset 0 length 520: 4 00000100:00100000:0.0:1767319636.883782:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319636.891403:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636224 00000100:00100000:1.0:1767319636.891420:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636224 00000100:00100000:1.0:1767319636.891774:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636096 00000100:00100000:1.0:1767319636.891779:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636096 00000100:00100000:1.0:1767319636.891810:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1234 00000100:00100000:1.0:1767319636.891821:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915636224:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767319636.892001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319636.892035:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1626 00000100:00100000:0.0:1767319636.892041:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636096:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319636.892063:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319636.892148:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915636224:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 328us (8437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319636.892156:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1234 00000100:00100000:0.0:1767319636.899806:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636096:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 7751us (16010us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319636.899822:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1626 00010000:00010000:2.0:1767319637.124842:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319638.148382:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319638.470012:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982866176:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319638.470096:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319638.470931:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866176 00000100:00100000:2.0:1767319638.470941:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866176 00000100:00100000:2.0:1767319638.471008:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836 00000100:00100000:2.0:1767319638.471018:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982866176:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319638.471060:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319638.471118:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982866176:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (1022us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319638.471126:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836 00000100:00080000:1.0:1767319638.471190:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319638.471240:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982866176:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319638.473981:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982866304:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319638.474060:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319638.474924:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866304 00000100:00100000:1.0:1767319638.474935:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866304 00000100:00100000:1.0:1767319638.475033:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 837 00000100:00100000:1.0:1767319638.475053:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982866304:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319638.475094:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767319638.475200:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319638.476551:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982866304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1504us (2486us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319638.476562:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 837 00000100:00100000:3.0:1767319638.477937:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982866304:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319639.172577:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319640.004878:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.004889:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319640.005013:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319614/real 1767319614] req@ffff9346100fdf80 x1853163982859904/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319639 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319640.005080:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100fdf80 x1853163982859904/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319639 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319640.005101:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319640.005124:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319640.005129:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982859904:0@lo:250:kworker.0 00000100:00080000:3.0:1767319640.005383:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.005387:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319640.005390:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319640.005395:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4883 00000100:00080000:3.0:1767319640.005399:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319640.005405:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319640.005424:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e680 x1853163982866560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319640.005451:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.005454:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319640.005455:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319640.005459:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4883 00000100:00080000:3.0:1767319640.005461:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319640.005465:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319640.005474:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d880 x1853163982866816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319640.005483:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.005485:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319640.005487:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319640.005490:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4883 00000100:00080000:3.0:1767319640.005492:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319640.005495:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319640.005506:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911180 x1853163982866944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319640.005533:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.005536:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319640.005538:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319640.005541:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4883 00000100:00080000:3.0:1767319640.005543:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319640.005546:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319640.005554:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913b80 x1853163982867328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319640.005567:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319640.005570:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319640.005571:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319640.005574:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4862 00000100:00080000:3.0:1767319640.005576:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319640.005579:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319640.005587:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912a00 x1853163982867456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319640.005611:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982867200:0@lo:400:kworker.0 00000100:00100000:3.0:1767319640.005682:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319640.005738:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319640.005744:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982867072:0@lo:400:kworker.0 00000100:00100000:3.0:1767319640.005764:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319640.005792:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867200 00000100:00100000:3.0:1767319640.005800:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867200 00000100:00100000:3.0:1767319640.005846:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867072 00000100:00100000:3.0:1767319640.005849:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867072 00000100:00100000:3.0:1767319640.005870:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 775 00000100:00100000:3.0:1767319640.005879:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982867200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319640.006031:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982866560:0@lo:38:kworker.0 00000100:00100000:2.0:1767319640.006073:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319640.006096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982866816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319640.006115:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319640.006136:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982866944:0@lo:38:kworker.0 00000100:00100000:2.0:1767319640.006156:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319640.006177:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982867328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319640.006196:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319640.006212:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982867456:0@lo:250:kworker.0 00000400:00000100:2.0:1767319640.006228:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982867456 offset 0 length 520: 4 00000100:00100000:2.0:1767319640.006256:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866560 00000100:00100000:2.0:1767319640.006259:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866560 00000100:00100000:2.0:1767319640.006309:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866816 00000100:00100000:2.0:1767319640.006312:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866816 00000100:00100000:2.0:1767319640.006325:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867328 00000100:00100000:2.0:1767319640.006327:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867328 00000100:00100000:2.0:1767319640.006338:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1627 00000100:00100000:2.0:1767319640.006343:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982866560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319640.006361:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319640.006398:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982866560:12345-0@lo:38:kworker.0 Request processed in 53us (324us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319640.006404:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1627 00000100:00100000:2.0:1767319640.006418:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1628 00000100:00100000:2.0:1767319640.006422:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982866816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319640.006431:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319640.006453:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982866816:12345-0@lo:38:kworker.0 Request processed in 29us (337us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319640.006459:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1628 00000100:00100000:2.0:1767319640.006469:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1629 00000100:00100000:2.0:1767319640.006473:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982867328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319640.006482:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319640.006503:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982867328:12345-0@lo:38:kworker.0 Request processed in 28us (305us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319640.006508:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1629 00000100:00100000:2.0:1767319640.006526:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866944 00000100:00100000:2.0:1767319640.006529:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866944 00000100:00100000:2.0:1767319640.006544:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 432 00000100:00100000:2.0:1767319640.006548:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982866944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319640.006556:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319640.006579:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982866944:12345-0@lo:38:kworker.0 Request processed in 29us (422us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319640.006584:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 432 00000100:00080000:2.0:1767319640.006622:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319640.006626:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319640.006629:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982866560:0@lo:38:kworker.0 00000100:00080000:2.0:1767319640.006636:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319640.006638:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319640.006640:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982866816:0@lo:38:kworker.0 00000100:00080000:2.0:1767319640.006647:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319640.006649:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319640.006652:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982866944:0@lo:38:kworker.0 00000100:00080000:2.0:1767319640.006657:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319640.006659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319640.006661:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982867328:0@lo:38:kworker.0 00000100:00100000:2.0:1767319640.006701:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 776 00000100:00100000:2.0:1767319640.006709:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982867072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319640.006745:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982867072:12345-0@lo:400:kworker.0 Request processed in 37us (979us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319640.006750:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 776 00000100:00100000:2.0:1767319640.006803:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982867072:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.007190:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982866432:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.007223:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319640.007258:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319640.007266:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982866688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.007303:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319640.007328:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319640.007332:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319640.007343:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866432 00000100:00100000:0.0:1767319640.007347:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866432 00000100:00100000:0.0:1767319640.007382:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982866688 00000100:00100000:0.0:1767319640.007386:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982866688 00000100:00100000:0.0:1767319640.007403:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767319640.007409:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982866432:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.007444:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982866432:12345-0@lo:400:kworker.0 Request processed in 35us (220us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319640.007450:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767319640.007463:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236 00000100:00100000:0.0:1767319640.007470:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982866688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319640.007494:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982866688:12345-0@lo:400:kworker.0 Request processed in 26us (190us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319640.007499:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236 00000100:00080000:0.0:1767319640.007518:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319640.007537:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ea00 x1853163982866688/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319656 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319640.007548:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982866688:0@lo:400:kworker.0 00010000:00080000:0.0:1767319640.007556:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d180 x1853163982866432/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319656 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319640.007564:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982866432:0@lo:400:kworker.0 00000100:00100000:3.0:1767319640.007782:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982867200:12345-0@lo:400:kworker.0 Request processed in 1903us (2099us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319640.007790:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 775 00000100:00100000:2.0:1767319640.007972:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982867200:0@lo:400:kworker.0 00010000:00010000:2.0:1767319640.198850:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319641.220438:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319641.995643:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319641.995988:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636480 00000100:00100000:1.0:1767319641.996001:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636480 00000100:00100000:1.0:1767319641.996081:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1237 00000100:00100000:1.0:1767319641.996091:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915636480:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319641.996184:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915636480:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (552us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319641.996191:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1237 00000100:00080000:0.0:1767319641.996558:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319641.998106:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319641.998244:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636352 00000100:00100000:3.0:1767319641.998254:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636352 00000100:00100000:3.0:1767319641.998325:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1630 00000100:00100000:3.0:1767319641.998594:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915636352:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319641.998619:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319641.998696:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915636352:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 361us (597us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319641.998703:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1630 00010000:00010000:1.0:1767319642.244506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319643.268545:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319643.590031:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982867584:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319643.590116:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319643.590382:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982867712:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319643.591909:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319643.595874:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867584 00000100:00100000:3.0:1767319643.595890:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867584 00000100:00100000:3.0:1767319643.595978:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867712 00000100:00100000:3.0:1767319643.595983:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867712 00000100:00100000:3.0:1767319643.596005:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 838 00000100:00100000:3.0:1767319643.596014:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982867584:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319643.596057:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319643.596126:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982867584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (6008us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319643.596134:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 838 00000100:00100000:3.0:1767319643.596153:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839 00000100:00100000:3.0:1767319643.596159:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982867712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319643.596175:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319643.596206:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982867712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (4302us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319643.596211:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839 00000100:00080000:0.0:1767319643.596551:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319643.596557:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319643.596665:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982867712:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319643.596834:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982867584:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1767319644.292438:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319645.124443:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319645.124452:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319645.124505:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319645.124509:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319645.124512:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319645.124520:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4888 00000100:00080000:3.0:1767319645.124525:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319645.124537:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319645.124551:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982867840:0@lo:400:kworker.0 00000100:00080000:3.0:1767319645.124634:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319645.124662:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163982867968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319645.124674:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319645.124808:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867840 00000100:00100000:1.0:1767319645.124816:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867840 00000100:00100000:1.0:1767319645.124874:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1238 00000100:00080000:0.0:1767319645.124878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319645.124885:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982867840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319645.124899:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982867968:0@lo:38:kworker.0 00000100:00100000:2.0:1767319645.124953:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319645.124965:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982867840:12345-0@lo:400:kworker.0 Request processed in 81us (293us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319645.124973:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1238 00000100:00100000:1.0:1767319645.124993:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982867968 00000100:00100000:1.0:1767319645.124996:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982867968 00000100:00100000:1.0:1767319645.125010:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1631 00000100:00100000:1.0:1767319645.125014:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982867968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319645.125032:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319645.125082:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319645.125086:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319645.125088:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319645.125092:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4888 00000100:00080000:3.0:1767319645.125095:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319645.125099:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319645.125104:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982868096:0@lo:400:kworker.0 00000100:00100000:3.0:1767319645.125111:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5500 x1853163982868224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319645.125146:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767319645.125179:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015c00 x1853163982867840/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319661 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319645.125193:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982867840:0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.125224:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868096 00000100:00100000:0.0:1767319645.125228:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868096 00000100:00100000:0.0:1767319645.125260:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239 00000100:00100000:0.0:1767319645.125266:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982868096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319645.125331:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982868096:12345-0@lo:400:kworker.0 Request processed in 65us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319645.125337:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239 00000100:00080000:0.0:1767319645.125353:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319645.125376:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5180 x1853163982868096/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319661 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319645.125385:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982868096:0@lo:400:kworker.0 00000100:00100000:2.0:1767319645.130039:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982868224:0@lo:38:kworker.0 00000100:00100000:2.0:1767319645.130125:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.130253:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868224 00000100:00100000:0.0:1767319645.130263:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868224 00000100:00100000:0.0:1767319645.130334:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632 00000100:00100000:0.0:1767319645.130342:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982868224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319645.130365:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.130407:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982868224:12345-0@lo:38:kworker.0 Request processed in 66us (282us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.130414:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632 00000100:00100000:1.0:1767319645.133137:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982867968:12345-0@lo:38:kworker.0 Request processed in 8110us (8174us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319645.133153:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1631 00000100:00080000:2.0:1767319645.133249:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319645.133256:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319645.133261:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982867968:0@lo:38:kworker.0 00000100:00080000:3.0:1767319645.133282:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319645.133288:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319645.133292:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319645.133298:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4888 00000100:00080000:3.0:1767319645.133302:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319645.133308:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:2.0:1767319645.133334:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319645.133341:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319645.133344:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982868224:0@lo:38:kworker.0 00000100:00100000:3.0:1767319645.133349:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6d80 x1853163982868352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319645.133549:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982868480:0@lo:400:kworker.0 00000100:00100000:1.0:1767319645.133603:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319645.133937:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982868352:0@lo:38:kworker.0 00000100:00100000:2.0:1767319645.133969:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319645.134263:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868352 00000100:00100000:2.0:1767319645.134265:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868480 00000100:00100000:0.0:1767319645.134293:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868352 00000100:00100000:2.0:1767319645.134302:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868480 00000100:00100000:0.0:1767319645.134333:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 433 00000100:00100000:0.0:1767319645.134342:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982868352:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319645.134357:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 777 00000100:00100000:2.0:1767319645.134365:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982868480:12345-0@lo:400:kworker.0 00010000:00020000:0.0:1767319645.134375:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319645.134433:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982868352:12345-0@lo:38:kworker.0 Request processed in 84us (455us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319645.134441:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 433 00000100:00080000:0.0:1767319645.134471:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319645.142013:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319645.142025:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319645.142030:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319645.142038:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4888 00000100:00080000:3.0:1767319645.142042:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319645.142049:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319645.142090:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7480 x1853163982868736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319645.142135:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982868608:0@lo:400:kworker.0 00000100:00100000:3.0:1767319645.142199:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319645.142294:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982868736:0@lo:38:kworker.0 00000100:00100000:3.0:1767319645.142321:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319645.142356:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319645.142372:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319645.142376:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982868352:0@lo:38:kworker.0 00000100:00100000:1.0:1767319645.142509:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868608 00000100:00100000:1.0:1767319645.142515:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868608 00000100:00100000:1.0:1767319645.142579:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 778 00000100:00100000:1.0:1767319645.142589:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982868608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319645.142644:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982868608:12345-0@lo:400:kworker.0 Request processed in 56us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319645.142651:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 778 00000100:00100000:1.0:1767319645.142702:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982868608:0@lo:400:kworker.0 00000100:00100000:1.0:1767319645.142725:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868736 00000100:00100000:1.0:1767319645.142729:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868736 00000100:00100000:1.0:1767319645.142752:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1633 00000100:00100000:1.0:1767319645.142757:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982868736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319645.142776:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319645.142809:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982868736:12345-0@lo:38:kworker.0 Request processed in 51us (487us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319645.142816:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1633 00000100:00100000:2.0:1767319645.142955:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982868480:12345-0@lo:400:kworker.0 Request processed in 8589us (9349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319645.142965:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 777 00000100:00080000:3.0:1767319645.142981:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319645.143003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319645.143008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319645.143011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982868736:0@lo:38:kworker.0 00000100:00100000:0.0:1767319645.143487:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982868480:0@lo:400:kworker.0 00010000:00010000:1.0:1767319645.317580:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319646.341107:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319647.116772:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319647.116822:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319647.116945:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636736 00000100:00100000:1.0:1767319647.116956:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636736 00000100:00080000:3.0:1767319647.117086:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319647.117156:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636608 00000100:00100000:1.0:1767319647.117162:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636608 00000100:00100000:1.0:1767319647.117190:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1240 00000100:00100000:1.0:1767319647.117198:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915636736:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319647.117378:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1634 00000100:00100000:3.0:1767319647.117385:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636608:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319647.117405:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319647.117526:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915636736:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 328us (763us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319647.117534:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1240 00000100:00100000:3.0:1767319647.119124:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636608:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 1729us (2293us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319647.119136:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1634 00010000:00010000:0.0:1767319647.364464:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319648.388499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319648.708935:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982868864:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319648.709025:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319648.709132:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982868992:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319648.709168:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319648.718335:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868864 00000100:00100000:0.0:1767319648.718352:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868864 00000100:00100000:0.0:1767319648.718449:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982868992 00000100:00100000:0.0:1767319648.718453:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982868992 00000100:00100000:0.0:1767319648.718474:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 840 00000100:00100000:0.0:1767319648.718483:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982868864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319648.718517:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319648.718620:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982868864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 137us (9594us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319648.718628:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 840 00000100:00100000:0.0:1767319648.718645:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 841 00000100:00100000:0.0:1767319648.718651:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982868992:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319648.718667:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319648.718698:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982868992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (9530us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319648.718703:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 841 00000100:00100000:0.0:1767319648.718770:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982868864:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319648.718920:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319648.718925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319648.724016:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982868992:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319649.414601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319650.247617:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319650.247627:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319650.247688:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319650.247692:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319650.247696:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319650.247704:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4893 00000100:00080000:2.0:1767319650.247709:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319650.247715:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319650.247737:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e680 x1853163982869248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319650.247769:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767319650.247769:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319650.247782:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319650.247784:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319650.247788:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4893 00000100:00080000:2.0:1767319650.247791:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319650.247794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982869376:0@lo:400:kworker.0 00000100:00080000:2.0:1767319650.247797:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319650.247825:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988e00 x1853163982869504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319650.247847:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319650.247850:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319650.247852:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319650.247855:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4893 00000100:00080000:2.0:1767319650.247858:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319650.247861:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319650.247879:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988000 x1853163982869632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319650.247896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319650.247918:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319650.247921:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319650.247924:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319650.247929:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4893 00000100:00080000:2.0:1767319650.247931:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319650.247935:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319650.247943:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982869120:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.247948:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989180 x1853163982870016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319650.247973:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.247990:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982869248:0@lo:38:kworker.0 00000100:00100000:1.0:1767319650.248014:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982869888:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.248023:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.248051:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982869504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319650.248058:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.248079:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319650.248090:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319650.248099:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982869760:0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.248103:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982869632:0@lo:38:kworker.0 00000100:00100000:1.0:1767319650.248135:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319650.248143:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319650.248173:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869888 00000100:00100000:1.0:1767319650.248181:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869888 00000100:00100000:2.0:1767319650.248195:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982870016:0@lo:38:kworker.0 00000100:00100000:2.0:1767319650.248225:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319650.248233:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869760 00000100:00100000:1.0:1767319650.248238:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869760 00000100:00100000:2.0:1767319650.248262:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869376 00000100:00100000:2.0:1767319650.248267:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869376 00000100:00100000:1.0:1767319650.248298:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 779 00000100:00100000:1.0:1767319650.248310:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982869888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319650.248316:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869120 00000100:00100000:2.0:1767319650.248321:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869120 00000100:00100000:2.0:1767319650.248336:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869248 00000100:00100000:2.0:1767319650.248340:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869248 00000100:00100000:2.0:1767319650.248352:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869504 00000100:00100000:2.0:1767319650.248354:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869504 00000100:00100000:2.0:1767319650.248366:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870016 00000100:00100000:2.0:1767319650.248369:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870016 00000100:00100000:1.0:1767319650.248370:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982869888:12345-0@lo:400:kworker.0 Request processed in 62us (311us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319650.248380:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 779 00000100:00100000:2.0:1767319650.248384:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241 00000100:00100000:1.0:1767319650.248403:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 780 00000100:00100000:2.0:1767319650.248407:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982869376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.248415:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982869760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.248451:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982869760:12345-0@lo:400:kworker.0 Request processed in 40us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319650.248459:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 780 00000100:00100000:2.0:1767319650.248461:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982869376:12345-0@lo:400:kworker.0 Request processed in 56us (564us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319650.248470:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241 00000100:00100000:2.0:1767319650.248490:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1242 00000100:00100000:2.0:1767319650.248498:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982869120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.248506:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982869760:0@lo:400:kworker.0 00000100:00100000:1.0:1767319650.248521:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982869888:0@lo:400:kworker.0 00000100:00100000:3.0:1767319650.248736:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1635 00000100:00100000:3.0:1767319650.248743:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982869248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319650.248760:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319650.248820:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982869120:12345-0@lo:400:kworker.0 Request processed in 322us (848us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319650.248836:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1242 00000100:00100000:2.0:1767319650.248850:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1636 00000100:00100000:2.0:1767319650.248854:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982869504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319650.248864:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319650.248894:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982869504:12345-0@lo:38:kworker.0 Request processed in 38us (815us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319650.248900:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1636 00000100:00100000:2.0:1767319650.248910:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1637 00000100:00100000:2.0:1767319650.248914:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982870016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319650.248924:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319650.248944:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982870016:12345-0@lo:38:kworker.0 Request processed in 28us (720us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319650.248949:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1637 00000100:00080000:2.0:1767319650.248975:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319650.248980:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319650.248983:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982869504:0@lo:38:kworker.0 00000100:00080000:2.0:1767319650.248992:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319650.248994:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319650.248997:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982870016:0@lo:38:kworker.0 00000100:00080000:2.0:1767319650.249026:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319650.249030:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319650.249032:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319650.250330:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982869632 00000100:00100000:0.0:1767319650.250341:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982869632 00000100:00100000:0.0:1767319650.250398:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 434 00000100:00100000:0.0:1767319650.250405:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982869632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319650.250422:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319650.250481:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982869632:12345-0@lo:38:kworker.0 Request processed in 75us (2340us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319650.250488:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 434 00010000:00080000:1.0:1767319650.250540:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ea00 x1853163982869376/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319666 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319650.250553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982869376:0@lo:400:kworker.0 00010000:00080000:1.0:1767319650.250566:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d880 x1853163982869120/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319666 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319650.250573:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982869120:0@lo:400:kworker.0 00000100:00080000:2.0:1767319650.252179:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319650.252190:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319650.252195:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982869632:0@lo:38:kworker.0 00000100:00100000:3.0:1767319650.260148:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982869248:12345-0@lo:38:kworker.0 Request processed in 11394us (12115us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319650.260164:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1635 00000100:00080000:0.0:1767319650.260303:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319650.260310:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319650.260314:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982869248:0@lo:38:kworker.0 00010000:00010000:0.0:1767319650.438069:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319651.461331:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319652.244259:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319652.244338:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636864 00000100:00100000:1.0:1767319652.244348:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636864 00000100:00100000:0.0:1767319652.244390:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319652.244396:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1638 00000100:00100000:1.0:1767319652.244406:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319652.244431:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319652.244440:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915636992 00000100:00100000:0.0:1767319652.244445:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915636992 00000100:00100000:0.0:1767319652.244498:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1243 00000100:00100000:1.0:1767319652.244498:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915636864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (248us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319652.244508:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915636992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319652.244513:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1638 00000100:00080000:2.0:1767319652.244542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319652.244554:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915636992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 48us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319652.244561:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1243 00010000:00010000:2.0:1767319652.486704:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1767319653.508491:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319653.828531:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982870144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319653.828607:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319653.828774:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870144 00000100:00100000:0.0:1767319653.828783:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870144 00000100:00100000:0.0:1767319653.828854:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842 00000100:00100000:0.0:1767319653.828866:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982870144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319653.828909:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319653.828953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982870272:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319653.828959:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982870144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319653.828969:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842 00000100:00100000:1.0:1767319653.828993:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319653.829021:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870272 00000100:00100000:0.0:1767319653.829026:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870272 00000100:00100000:0.0:1767319653.829057:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 843 00000100:00100000:0.0:1767319653.829063:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982870272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319653.829079:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319653.829120:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982870272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (128us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319653.829126:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 843 00000100:00100000:2.0:1767319653.829239:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982870272:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319653.829262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319653.829267:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319653.829325:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982870144:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319654.532558:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319655.364531:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.364542:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319655.364608:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.364612:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.364617:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.364626:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.364631:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.364637:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319655.364658:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982870528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319655.364678:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.364681:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.364682:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.364686:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.364688:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.364691:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319655.364702:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597b80 x1853163982870784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319655.364710:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.364713:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.364715:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.364718:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.364720:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.364724:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319655.364747:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596300 x1853163982870912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319655.364783:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982870656:0@lo:400:kworker.0 00000100:00080000:0.0:1767319655.364785:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319655.364792:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319655.364795:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319655.364799:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4898 00000100:00080000:0.0:1767319655.364801:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319655.364805:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319655.364817:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595880 x1853163982871296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319655.364842:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982871040:0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.364860:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319655.364933:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982870400:0@lo:400:kworker.0 00000100:00100000:2.0:1767319655.364989:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982870528:0@lo:38:kworker.0 00000100:00100000:0.0:1767319655.368372:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319655.368645:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319655.368659:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982871168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319655.368722:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319655.368776:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871040 00000100:00100000:0.0:1767319655.368783:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871040 00000100:00100000:0.0:1767319655.369117:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871168 00000100:00100000:0.0:1767319655.369122:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871168 00000100:00100000:0.0:1767319655.369152:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 781 00000100:00100000:0.0:1767319655.369160:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982871040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319655.369215:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982871040:12345-0@lo:400:kworker.0 Request processed in 56us (849us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319655.369223:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 781 00000100:00100000:0.0:1767319655.369238:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 782 00000100:00100000:0.0:1767319655.369245:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982871168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319655.369295:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982871168:12345-0@lo:400:kworker.0 Request processed in 50us (572us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319655.369301:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 782 00000100:00100000:0.0:1767319655.369330:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870656 00000100:00100000:0.0:1767319655.369334:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870656 00000100:00100000:0.0:1767319655.369361:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244 00000100:00100000:0.0:1767319655.369368:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982870656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319655.369401:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982870656:12345-0@lo:400:kworker.0 Request processed in 34us (4542us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319655.369420:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244 00000100:00100000:0.0:1767319655.369645:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982871168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319655.369658:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982871040:0@lo:400:kworker.0 00000100:00080000:0.0:1767319655.369864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319655.369868:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319655.369870:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:3.0:1767319655.370433:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594380 x1853163982870656/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319671 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319655.370446:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982870656:0@lo:400:kworker.0 00000100:00100000:1.0:1767319655.372318:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319655.372382:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870400 00000100:00100000:0.0:1767319655.372390:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870400 00000100:00100000:2.0:1767319655.372598:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319655.372858:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982870784:0@lo:38:kworker.0 00000100:00100000:2.0:1767319655.372909:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319655.372965:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319655.373146:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982870912:0@lo:38:kworker.0 00000100:00100000:2.0:1767319655.373174:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319655.373197:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982871296:0@lo:38:kworker.0 00000100:00100000:2.0:1767319655.373218:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319655.373357:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870528 00000100:00100000:3.0:1767319655.373363:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870528 00000100:00100000:3.0:1767319655.373383:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870784 00000100:00100000:3.0:1767319655.373385:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870784 00000100:00100000:3.0:1767319655.373395:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245 00000100:00100000:3.0:1767319655.373404:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982870400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.373479:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982870400:12345-0@lo:400:kworker.0 Request processed in 77us (1420us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319655.373487:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245 00000100:00100000:3.0:1767319655.373504:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1639 00000100:00100000:3.0:1767319655.373508:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982870528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.373526:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.373554:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982870528:12345-0@lo:38:kworker.0 Request processed in 44us (956us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.373560:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1639 00000100:00100000:3.0:1767319655.373572:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1640 00000100:00100000:3.0:1767319655.373576:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982870784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.373585:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.373605:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982870784:12345-0@lo:38:kworker.0 Request processed in 28us (696us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.373610:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1640 00000100:00100000:3.0:1767319655.373650:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982870912 00000100:00100000:3.0:1767319655.373654:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982870912 00000100:00100000:3.0:1767319655.373678:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 435 00000100:00100000:3.0:1767319655.373683:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982870912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.373691:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319655.373718:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982870912:12345-0@lo:38:kworker.0 Request processed in 34us (544us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.373724:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 435 00000100:00080000:2.0:1767319655.374265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319655.374286:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319655.374291:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982870528:0@lo:38:kworker.0 00000100:00080000:2.0:1767319655.374302:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319655.374306:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319655.374309:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982870784:0@lo:38:kworker.0 00000100:00080000:2.0:1767319655.374318:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319655.374322:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319655.374324:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982870912:0@lo:38:kworker.0 00000100:00100000:3.0:1767319655.374597:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871296 00000100:00100000:3.0:1767319655.374601:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871296 00000100:00100000:3.0:1767319655.374632:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1641 00000100:00100000:3.0:1767319655.374637:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982871296:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319655.374646:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:1.0:1767319655.374813:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320380 x1853163982870400/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319671 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319655.374826:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982870400:0@lo:400:kworker.0 00000100:00100000:3.0:1767319655.375000:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982871296:12345-0@lo:38:kworker.0 Request processed in 360us (1779us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319655.375006:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1641 00000100:00080000:2.0:1767319655.375057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319655.375062:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319655.375065:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982871296:0@lo:38:kworker.0 00010000:00010000:2.0:1767319655.558805:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319656.580573:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319657.360422:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319657.360483:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319657.360524:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637120 00000100:00100000:2.0:1767319657.360749:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637248 00000100:00100000:2.0:1767319657.360761:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637248 00000100:00100000:2.0:1767319657.360857:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1246 00000100:00100000:2.0:1767319657.360867:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915637248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319657.360948:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915637248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (464us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319657.360955:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1246 00000100:00100000:3.0:1767319657.361087:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637120 00000100:00100000:3.0:1767319657.361129:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1642 00000100:00100000:3.0:1767319657.361135:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915637120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319657.361164:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319657.361199:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915637120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (790us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319657.361206:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1642 00000100:00080000:1.0:1767319657.361423:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319657.604474:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319658.629893:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319658.948580:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319658.948604:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982871424:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319658.948719:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319658.948974:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871424 00000100:00100000:2.0:1767319658.948983:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871424 00000100:00100000:2.0:1767319658.949046:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 844 00000100:00100000:2.0:1767319658.949055:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982871424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319658.949084:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319658.949133:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982871424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (413us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319658.949140:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 844 00000100:00100000:2.0:1767319658.949184:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982871424:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319658.949250:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319658.949597:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982871552:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319658.949624:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319658.949894:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871552 00000100:00100000:1.0:1767319658.949900:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871552 00000100:00100000:1.0:1767319658.950047:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 845 00000100:00100000:1.0:1767319658.950055:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982871552:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319658.950076:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319658.950119:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982871552:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (494us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319658.950126:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 845 00000100:00100000:2.0:1767319658.950185:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982871552:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319658.950238:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319659.652799:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319660.486978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319660.486989:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319660.487064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319660.487068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319660.487071:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319660.487080:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4903 00000100:00080000:0.0:1767319660.487085:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319660.487092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319660.487113:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597b80 x1853163982871808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319660.487147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319660.487150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319660.487152:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319660.487155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4903 00000100:00080000:0.0:1767319660.487158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319660.487161:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319660.487172:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595880 x1853163982872064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319660.487180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319660.487183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319660.487184:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319660.487187:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4903 00000100:00080000:0.0:1767319660.487189:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319660.487193:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319660.487208:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2d80 x1853163982872192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319660.487238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319660.487242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319660.487243:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319660.487247:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4903 00000100:00080000:0.0:1767319660.487249:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319660.487252:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319660.487262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163982872576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319660.487311:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982872320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.487392:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319660.487458:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319660.487464:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982872448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.487485:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319660.487515:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872320 00000100:00100000:0.0:1767319660.487522:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872320 00000100:00100000:0.0:1767319660.487580:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872448 00000100:00100000:0.0:1767319660.487583:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872448 00000100:00100000:0.0:1767319660.487603:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 783 00000100:00100000:0.0:1767319660.487610:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982872320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.487664:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982872320:12345-0@lo:400:kworker.0 Request processed in 54us (272us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319660.487671:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 783 00000100:00100000:0.0:1767319660.487685:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 784 00000100:00100000:0.0:1767319660.487690:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982872448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.487712:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982872448:12345-0@lo:400:kworker.0 Request processed in 22us (227us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319660.487717:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 784 00000100:00080000:0.0:1767319660.487732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319660.487736:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319660.487762:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982872448:0@lo:400:kworker.0 00000100:00100000:0.0:1767319660.487772:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982872320:0@lo:400:kworker.0 00000100:00100000:3.0:1767319660.488034:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982871808:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.488065:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.488096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982872064:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.488116:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.488134:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982872192:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.488152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.488169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982872576:0@lo:38:kworker.0 00000100:00100000:3.0:1767319660.488187:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319660.488210:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871808 00000100:00100000:3.0:1767319660.488214:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871808 00000100:00100000:3.0:1767319660.488234:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872064 00000100:00100000:3.0:1767319660.488236:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872064 00000100:00100000:3.0:1767319660.488248:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872576 00000100:00100000:3.0:1767319660.488250:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872576 00000100:00100000:3.0:1767319660.488260:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1643 00000100:00100000:3.0:1767319660.488265:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982871808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319660.488308:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319660.488340:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982871808:12345-0@lo:38:kworker.0 Request processed in 73us (274us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319660.488346:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1643 00000100:00100000:3.0:1767319660.488361:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1644 00000100:00100000:3.0:1767319660.488365:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982872064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319660.488387:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319660.488409:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982872064:12345-0@lo:38:kworker.0 Request processed in 43us (293us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319660.488415:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1644 00000100:00100000:3.0:1767319660.488425:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1645 00000100:00100000:3.0:1767319660.488428:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982872576:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319660.488439:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319660.488458:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982872576:12345-0@lo:38:kworker.0 Request processed in 28us (271us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319660.488463:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1645 00000100:00100000:3.0:1767319660.488480:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872192 00000100:00100000:3.0:1767319660.488484:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872192 00000100:00100000:3.0:1767319660.488502:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 436 00000100:00100000:3.0:1767319660.488506:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982872192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319660.488514:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319660.488538:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982872192:12345-0@lo:38:kworker.0 Request processed in 31us (385us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319660.488542:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 436 00000100:00080000:3.0:1767319660.488574:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319660.488579:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319660.488581:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982871808:0@lo:38:kworker.0 00000100:00080000:3.0:1767319660.488588:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319660.488592:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319660.488594:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982872064:0@lo:38:kworker.0 00000100:00080000:3.0:1767319660.488600:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319660.488603:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319660.488606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982872192:0@lo:38:kworker.0 00000100:00080000:3.0:1767319660.488611:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319660.488614:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319660.488617:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982872576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319660.489827:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982871680:0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.489883:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319660.489940:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319660.489949:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982871936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.489974:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319660.490008:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871680 00000100:00100000:2.0:1767319660.490013:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871680 00000100:00100000:2.0:1767319660.490071:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982871936 00000100:00100000:2.0:1767319660.490075:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982871936 00000100:00100000:2.0:1767319660.490107:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247 00000100:00100000:2.0:1767319660.490115:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982871680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.490166:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982871680:12345-0@lo:400:kworker.0 Request processed in 51us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319660.490173:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247 00000100:00100000:2.0:1767319660.490189:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248 00000100:00100000:2.0:1767319660.490195:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982871936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319660.490222:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982871936:12345-0@lo:400:kworker.0 Request processed in 27us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319660.490227:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248 00010000:00080000:2.0:1767319660.490261:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594380 x1853163982871936/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319676 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319660.490311:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982871936:0@lo:400:kworker.0 00010000:00080000:2.0:1767319660.490324:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596300 x1853163982871680/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319676 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319660.490333:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982871680:0@lo:400:kworker.0 00000100:00080000:0.0:1767319660.490968:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319660.676793:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319661.707839:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319662.476496:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319662.476880:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637632 00000100:00100000:0.0:1767319662.476892:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637632 00000100:00100000:0.0:1767319662.476983:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1249 00000100:00100000:0.0:1767319662.477137:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915637632:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319662.477223:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915637632:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (737us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319662.477231:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1249 00000100:00080000:2.0:1767319662.477915:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767319662.478558:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915637376 offset 0 length 520: 4 00000100:00100000:0.0:1767319662.478647:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319662.479004:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637504 00000100:00100000:2.0:1767319662.479013:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637504 00000100:00100000:2.0:1767319662.479052:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1646 00000100:00100000:2.0:1767319662.479058:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915637504:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319662.479079:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319662.479133:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915637504:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 75us (488us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319662.479138:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1646 00010000:00010000:2.0:1767319662.724502:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319663.752443:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319664.068555:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319664.068611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982872704:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319664.068684:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319664.068756:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872704 00000100:00100000:2.0:1767319664.068762:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872704 00000100:00100000:2.0:1767319664.068817:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 846 00000100:00100000:2.0:1767319664.068824:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982872704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319664.068851:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319664.068882:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982872704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (199us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319664.068887:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 846 00000100:00080000:2.0:1767319664.068903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319664.068929:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982872704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319664.070886:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319664.070900:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982872832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319664.070959:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319664.071250:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982872832 00000100:00100000:0.0:1767319664.071259:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982872832 00000100:00100000:0.0:1767319664.071341:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 847 00000100:00100000:0.0:1767319664.071350:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982872832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319664.071378:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319664.071431:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982872832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (477us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319664.071438:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 847 00000100:00100000:0.0:1767319664.071487:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982872832:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319664.072077:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319664.772606:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319665.605927:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319640/real 1767319640] req@ffff93460a912a00 x1853163982867456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319665 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319665.605965:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a912a00 x1853163982867456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319665 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319665.605988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319665.605997:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319665.606001:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982867456:0@lo:250:kworker.0 00000100:00080000:3.0:1767319665.608417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319665.608427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319665.608431:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319665.608451:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4888 00000100:00080000:3.0:1767319665.608456:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319665.608463:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319665.608499:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581180 x1853163982872960/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319665.608594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319665.608598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319665.608599:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319665.608717:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982873088:0@lo:400:kworker.0 00000100:00100000:1.0:1767319665.608789:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319665.608895:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873088 00000100:00100000:1.0:1767319665.608901:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873088 00000100:00100000:1.0:1767319665.608952:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1250 00000100:00100000:1.0:1767319665.608962:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982873088:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319665.609011:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982873088:12345-0@lo:400:kworker.0 Request processed in 51us (221us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319665.609019:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1250 00000100:00080000:1.0:1767319665.609041:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319665.609068:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582300 x1853163982873088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319681 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319665.609081:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982873088:0@lo:400:kworker.0 00000100:00100000:0.0:1767319665.609463:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982872960:0@lo:250:kworker.0 00000400:00000100:0.0:1767319665.609491:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982872960 offset 0 length 520: 4 00000100:00080000:3.0:1767319665.609861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4909 00000100:00080000:3.0:1767319665.609866:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319665.609871:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319665.609893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583b80 x1853163982873216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319665.609935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319665.609938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319665.609940:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319665.609944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4909 00000100:00080000:3.0:1767319665.609946:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319665.609950:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319665.609964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582680 x1853163982873472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319665.609979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319665.609982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319665.609984:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319665.609987:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4909 00000100:00080000:3.0:1767319665.609989:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319665.609992:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319665.610001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582d80 x1853163982873600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319665.610026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319665.610029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319665.610031:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319665.610034:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4909 00000100:00080000:3.0:1767319665.610037:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319665.610039:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319665.610049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581c00 x1853163982873984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319665.611319:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982873728:0@lo:400:kworker.0 00000100:00100000:0.0:1767319665.611380:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319665.611427:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319665.611434:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982873856:0@lo:400:kworker.0 00000100:00100000:0.0:1767319665.611465:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319665.611508:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873728 00000100:00100000:0.0:1767319665.611515:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873728 00000100:00100000:0.0:1767319665.611573:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873856 00000100:00100000:0.0:1767319665.611576:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873856 00000100:00100000:0.0:1767319665.611599:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 785 00000100:00100000:0.0:1767319665.612434:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982873728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319665.612524:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982873728:12345-0@lo:400:kworker.0 Request processed in 916us (1143us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319665.612533:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 785 00000100:00100000:0.0:1767319665.612590:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 786 00000100:00100000:0.0:1767319665.612598:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982873856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319665.612644:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982873856:12345-0@lo:400:kworker.0 Request processed in 47us (1178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319665.612651:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 786 00000100:00100000:2.0:1767319665.613061:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982873344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.613102:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.613151:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873344 00000100:00100000:2.0:1767319665.613157:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873344 00000100:00100000:2.0:1767319665.622800:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251 00000100:00100000:2.0:1767319665.622814:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982873344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.622906:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982873344:12345-0@lo:400:kworker.0 Request processed in 95us (9803us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319665.622915:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251 00010000:00080000:2.0:1767319665.622976:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580000 x1853163982873344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319681 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319665.622991:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982873344:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.623026:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982873216:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.623064:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.623087:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982873472:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.623110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.623126:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982873600:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.623151:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.623184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982873984:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.623207:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319665.623237:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873216 00000100:00100000:2.0:1767319665.623243:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873216 00000100:00100000:2.0:1767319665.623262:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873472 00000100:00100000:2.0:1767319665.623266:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873472 00000100:00100000:2.0:1767319665.623293:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873984 00000100:00100000:2.0:1767319665.623297:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873984 00000100:00100000:2.0:1767319665.623307:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1647 00000100:00100000:2.0:1767319665.623311:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.623330:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.623364:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873216:12345-0@lo:38:kworker.0 Request processed in 52us (300us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.623384:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1647 00000100:00100000:2.0:1767319665.623396:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1648 00000100:00100000:2.0:1767319665.623400:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.623408:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.623432:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873472:12345-0@lo:38:kworker.0 Request processed in 31us (321us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.623949:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1648 00000100:00100000:2.0:1767319665.623968:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1649 00000100:00100000:2.0:1767319665.623972:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319665.623984:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319665.624017:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982873984:12345-0@lo:38:kworker.0 Request processed in 43us (808us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319665.624022:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1649 00000100:00080000:2.0:1767319665.624064:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.624069:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.624072:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982873216:0@lo:38:kworker.0 00000100:00080000:2.0:1767319665.624082:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.624085:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.624088:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982873472:0@lo:38:kworker.0 00000100:00080000:2.0:1767319665.624095:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.624098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.624101:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982873984:0@lo:38:kworker.0 00000100:00100000:2.0:1767319665.624156:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982873856:0@lo:400:kworker.0 00000100:00100000:2.0:1767319665.624179:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982873728:0@lo:400:kworker.0 00000100:00100000:3.0:1767319665.628953:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982873600 00000100:00100000:3.0:1767319665.628972:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982873600 00000100:00100000:3.0:1767319665.629043:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 437 00000100:00100000:3.0:1767319665.629051:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982873600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319665.629076:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319665.630653:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319665.630677:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319665.630683:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982873600:0@lo:38:kworker.0 00000100:00100000:3.0:1767319665.632923:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982873600:12345-0@lo:38:kworker.0 Request processed in 3858us (9757us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319665.632941:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 437 00000100:00080000:1.0:1767319665.634323:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319665.796519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319666.820482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319667.597429:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319667.597877:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637888 00000100:00100000:2.0:1767319667.597899:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637888 00000100:00100000:2.0:1767319667.597983:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1252 00000100:00100000:2.0:1767319667.597993:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915637888:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319667.598071:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915637888:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (651us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319667.598084:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1252 00000100:00100000:1.0:1767319667.600657:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319667.600849:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915637760 00000100:00100000:2.0:1767319667.600859:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915637760 00000100:00100000:2.0:1767319667.600911:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1650 00000100:00100000:2.0:1767319667.600919:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915637760:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319667.600943:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319667.601001:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915637760:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 80us (351us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319667.601009:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1650 00000100:00080000:3.0:1767319667.602091:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319667.847266:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319668.868777:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319669.188816:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982874112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319669.188913:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319669.188987:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874112 00000100:00100000:3.0:1767319669.188998:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874112 00000100:00100000:3.0:1767319669.189098:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 848 00000100:00100000:3.0:1767319669.189112:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982874112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319669.189154:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319669.189218:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982874240:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319669.189228:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982874112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (313us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319669.189238:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 848 00000100:00100000:0.0:1767319669.189290:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319669.189313:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874240 00000100:00100000:3.0:1767319669.189318:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874240 00000100:00100000:3.0:1767319669.189358:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 849 00000100:00100000:3.0:1767319669.189365:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982874240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319669.189383:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319669.189415:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982874240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (145us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319669.189422:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 849 00000100:00100000:0.0:1767319669.189611:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982874240:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319669.192698:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319669.192713:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319669.192819:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982874112:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319669.892567:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319670.724485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319670.724500:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319670.724645:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767319670.724670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319670.724675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319670.724680:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319670.724692:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4914 00000100:00080000:3.0:1767319670.724697:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319670.724706:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319670.724737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163982874496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319670.724857:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982874496:0@lo:38:kworker.0 00000100:00100000:1.0:1767319670.724953:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319670.725042:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874496 00000100:00100000:1.0:1767319670.725051:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874496 00000100:00100000:1.0:1767319670.725095:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1651 00000100:00100000:1.0:1767319670.725103:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982874496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319670.725128:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319670.725179:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982874496:12345-0@lo:38:kworker.0 Request processed in 75us (226us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319670.725187:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1651 00000100:00080000:1.0:1767319670.725235:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319670.725240:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319670.725243:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982874496:0@lo:38:kworker.0 00000100:00100000:0.0:1767319670.725638:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982874624:0@lo:400:kworker.0 00000100:00100000:0.0:1767319670.725678:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319670.725758:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982874368:0@lo:400:kworker.0 00000100:00100000:0.0:1767319670.725783:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319670.725849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319670.725854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319670.725858:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319670.725862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4914 00000100:00080000:3.0:1767319670.725865:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319670.725871:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319670.725893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163982874752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319670.725912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319670.725915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319670.725917:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319670.725920:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4914 00000100:00080000:3.0:1767319670.725923:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319670.725926:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319670.725938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582d80 x1853163982874880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319670.725973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319670.725976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319670.725978:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319670.725994:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4914 00000100:00080000:3.0:1767319670.725997:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319670.726001:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319670.726012:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581c00 x1853163982875264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319670.726042:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982875008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726091:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319670.726116:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319670.726122:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982875136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726144:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319670.726329:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875008 00000100:00100000:3.0:1767319670.726336:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875008 00000100:00100000:3.0:1767319670.726381:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875136 00000100:00100000:3.0:1767319670.726383:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875136 00000100:00100000:3.0:1767319670.726404:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 787 00000100:00100000:3.0:1767319670.726412:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982875008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726486:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982875008:12345-0@lo:400:kworker.0 Request processed in 74us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319670.726494:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 787 00000100:00100000:3.0:1767319670.726511:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 788 00000100:00100000:3.0:1767319670.726518:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982875136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726546:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982875136:12345-0@lo:400:kworker.0 Request processed in 29us (402us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319670.726552:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 788 00000100:00100000:3.0:1767319670.726589:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982875136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726599:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982875008:0@lo:400:kworker.0 00000100:00100000:3.0:1767319670.726661:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982874752:0@lo:38:kworker.0 00000100:00100000:0.0:1767319670.727249:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319670.727311:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874624 00000100:00100000:1.0:1767319670.727317:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874624 00000100:00100000:1.0:1767319670.727365:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874368 00000100:00100000:1.0:1767319670.727369:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874368 00000100:00100000:2.0:1767319670.727389:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253 00000100:00100000:2.0:1767319670.727403:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982874624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319670.727469:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982874624:12345-0@lo:400:kworker.0 Request processed in 70us (1790us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319670.727515:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1254 00000100:00100000:0.0:1767319670.727521:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982874880:0@lo:38:kworker.0 00000100:00100000:1.0:1767319670.727523:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982874368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319670.727557:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982874368:12345-0@lo:400:kworker.0 Request processed in 36us (1773us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319670.727562:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319670.727564:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1254 00000100:00100000:1.0:1767319670.727578:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874752 00000100:00100000:0.0:1767319670.727581:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982875264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319670.727581:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874752 00000100:00100000:1.0:1767319670.727593:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652 00000100:00100000:1.0:1767319670.727597:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982874752:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767319670.727602:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767319670.727618:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:0.0:1767319670.727633:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581500 x1853163982874624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319686 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319670.727650:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982874752:12345-0@lo:38:kworker.0 Request processed in 51us (401us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319670.727657:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982874624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319670.727661:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652 00010000:00080000:0.0:1767319670.727669:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582a00 x1853163982874368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319686 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319670.727669:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875264 00000100:00100000:1.0:1767319670.727672:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875264 00000100:00100000:0.0:1767319670.727679:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982874368:0@lo:400:kworker.0 00000100:00100000:1.0:1767319670.727682:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1653 00000100:00100000:1.0:1767319670.727686:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982875264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319670.727695:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319670.727705:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319670.727710:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319670.727713:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982874752:0@lo:38:kworker.0 00000100:00100000:1.0:1767319670.727716:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982875264:12345-0@lo:38:kworker.0 Request processed in 29us (112us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319670.727721:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767319670.727721:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1653 00000100:00080000:0.0:1767319670.727724:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319670.727726:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982875264:0@lo:38:kworker.0 00000100:00080000:1.0:1767319670.727744:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319670.727946:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982874880 00000100:00100000:0.0:1767319670.727951:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982874880 00000100:00100000:0.0:1767319670.727985:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 438 00000100:00100000:0.0:1767319670.727990:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982874880:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319670.728001:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319670.728032:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982874880:12345-0@lo:38:kworker.0 Request processed in 41us (469us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319670.728042:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 438 00000100:00080000:0.0:1767319670.728076:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319670.728081:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319670.728084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982874880:0@lo:38:kworker.0 00000100:00100000:2.0:1767319670.733015:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253 00010000:00010000:2.0:1767319670.916555:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319671.940560:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319672.719228:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319672.719315:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638144 00000100:00100000:2.0:1767319672.719329:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638144 00000100:00100000:1.0:1767319672.719361:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319672.719434:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638016 00000100:00100000:2.0:1767319672.719438:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638016 00000100:00100000:1.0:1767319672.719447:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1255 00000100:00100000:1.0:1767319672.719462:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915638144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319672.719503:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319672.719573:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915638144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 114us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319672.725911:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1255 00000100:00100000:2.0:1767319672.733225:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1654 00000100:00100000:2.0:1767319672.733246:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638016:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319672.733293:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319672.733428:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 185us (14066us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319672.733437:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1654 00010000:00010000:1.0:1767319672.964690:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319673.988649:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319674.308730:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319674.308764:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982875392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319674.308864:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319674.309319:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875392 00000100:00100000:0.0:1767319674.309337:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875392 00000100:00100000:0.0:1767319674.309455:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 850 00000100:00100000:0.0:1767319674.309468:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982875392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319674.309518:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319674.309596:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982875392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 130us (731us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319674.309604:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319674.309606:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 850 00000100:00100000:1.0:1767319674.310028:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982875392:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319674.311948:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982875520:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319674.312064:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319674.312213:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875520 00000100:00100000:2.0:1767319674.312225:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875520 00000100:00100000:2.0:1767319674.312371:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 851 00000100:00080000:0.0:1767319674.312541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319674.312635:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982875520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319674.313256:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319674.313414:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982875520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1031us (1350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319674.313424:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 851 00000100:00100000:1.0:1767319674.313738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982875520:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319675.012768:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319675.844666:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319675.844680:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319675.845412:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982875648:0@lo:400:kworker.0 00000100:00100000:3.0:1767319675.845511:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319675.845682:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319675.845687:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319675.845691:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319675.845697:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4919 00000100:00080000:0.0:1767319675.845700:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319675.846038:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875648 00000100:00100000:1.0:1767319675.846054:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875648 00000100:00100000:1.0:1767319675.846156:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256 00000100:00100000:1.0:1767319675.846178:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982875648:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767319675.846289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319675.846309:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319675.846343:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cf800 x1853163982875776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319675.846426:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982875776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319675.846480:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319675.846531:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875776 00000100:00100000:2.0:1767319675.846535:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875776 00000100:00100000:2.0:1767319675.846560:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1655 00000100:00100000:2.0:1767319675.846565:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982875776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319675.846587:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319675.846619:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982875776:12345-0@lo:38:kworker.0 Request processed in 53us (138us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319675.846625:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1655 00000100:00080000:2.0:1767319675.846663:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319675.846667:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319675.846671:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982875776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319675.846707:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982875904:0@lo:400:kworker.0 00000100:00100000:2.0:1767319675.846730:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319675.846753:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982875904 00000100:00100000:2.0:1767319675.846757:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982875904 00000100:00100000:2.0:1767319675.846782:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257 00000100:00100000:2.0:1767319675.846786:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982875904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319675.846817:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982875904:12345-0@lo:400:kworker.0 Request processed in 30us (85us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319675.846822:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257 00000100:00080000:2.0:1767319675.846834:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319675.846849:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c2300 x1853163982875904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319691 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319675.846860:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982875904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319675.853480:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982875648:12345-0@lo:400:kworker.0 Request processed in 7290us (7950us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319675.853502:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256 00010000:00080000:3.0:1767319675.854931:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cfb80 x1853163982875648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319691 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319675.854953:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982875648:0@lo:400:kworker.0 00000100:00080000:0.0:1767319675.856846:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319675.856856:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319675.856861:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319675.856868:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4919 00000100:00080000:0.0:1767319675.856873:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319675.856879:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319675.856921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0700 x1853163982876032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319675.856943:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319675.856947:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319675.856948:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319675.856952:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4919 00000100:00080000:0.0:1767319675.856954:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319675.856958:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319675.856970:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1f80 x1853163982876160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319675.857041:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319675.857046:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319675.857049:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319675.857049:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982876032:0@lo:38:kworker.0 00000100:00080000:0.0:1767319675.857061:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4919 00000100:00080000:0.0:1767319675.857065:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319675.857069:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319675.857157:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319675.857215:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982876160:0@lo:38:kworker.0 00000100:00100000:2.0:1767319675.857247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319675.857262:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982876288:0@lo:400:kworker.0 00000100:00100000:1.0:1767319675.857317:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319675.857333:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876160 00000100:00100000:1.0:1767319675.857343:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319675.857343:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876160 00000100:00100000:1.0:1767319675.857348:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982876416:0@lo:400:kworker.0 00000100:00100000:1.0:1767319675.857387:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319675.857399:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 439 00000100:00100000:2.0:1767319675.857409:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982876160:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319675.857412:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876032 00000100:00100000:1.0:1767319675.857416:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876032 00010000:00020000:2.0:1767319675.857437:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319675.857441:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656 00000100:00100000:1.0:1767319675.857445:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982876032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319675.857453:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319675.857485:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982876032:12345-0@lo:38:kworker.0 Request processed in 38us (328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319675.857493:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656 00000100:00100000:3.0:1767319675.857555:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876288 00000100:00100000:3.0:1767319675.857566:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876288 00000100:00100000:3.0:1767319675.857627:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876416 00000100:00100000:3.0:1767319675.857630:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876416 00000100:00100000:3.0:1767319675.857652:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 789 00000100:00100000:3.0:1767319675.857659:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982876288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319675.857763:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163982876544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319675.857822:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 790 00000100:00100000:0.0:1767319675.857831:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982876416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319675.857913:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982876288:12345-0@lo:400:kworker.0 Request processed in 253us (594us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319675.857920:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 789 00000100:00100000:3.0:1767319675.857968:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982876288:0@lo:400:kworker.0 00000100:00100000:3.0:1767319675.858015:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982876544:0@lo:38:kworker.0 00000100:00100000:3.0:1767319675.858049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319675.858091:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319675.858096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319675.858098:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982876032:0@lo:38:kworker.0 00000100:00080000:3.0:1767319675.858110:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319675.858113:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319675.858116:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982876160:0@lo:38:kworker.0 00000100:00100000:3.0:1767319675.858143:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876544 00000100:00100000:3.0:1767319675.858147:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876544 00000100:00100000:3.0:1767319675.858171:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1657 00000100:00100000:3.0:1767319675.858175:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982876544:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319675.858192:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319675.858225:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982876544:12345-0@lo:38:kworker.0 Request processed in 48us (175us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319675.858232:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1657 00000100:00080000:3.0:1767319675.858261:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319675.858265:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319675.858311:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982876544:0@lo:38:kworker.0 00000100:00100000:2.0:1767319675.859416:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982876160:12345-0@lo:38:kworker.0 Request processed in 1994us (2154us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319675.859434:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 439 00000100:00100000:0.0:1767319675.867995:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982876416:12345-0@lo:400:kworker.0 Request processed in 10155us (10596us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319675.868011:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 790 00000100:00100000:3.0:1767319675.868196:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982876416:0@lo:400:kworker.0 00010000:00010000:1.0:1767319676.036679:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319677.062794:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319677.837512:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319677.837586:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638272 00000100:00100000:1.0:1767319677.837600:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638272 00000100:00100000:0.0:1767319677.837636:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319677.837651:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638400 00000100:00100000:1.0:1767319677.837654:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638400 00000100:00100000:1.0:1767319677.837719:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1258 00000100:00100000:2.0:1767319677.837727:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1658 00000100:00100000:1.0:1767319677.837730:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915638400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319677.837734:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319677.837758:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319677.837820:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319677.837830:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (335us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319677.837838:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1658 00000100:00100000:1.0:1767319677.839600:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915638400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1855us (1947us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319677.839616:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1258 00010000:00010000:2.0:1767319678.084698:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319679.108892:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319679.429018:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982876672:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319679.429095:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319679.430003:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982876800:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319679.430081:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319679.430464:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876672 00000100:00100000:0.0:1767319679.430477:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876672 00000100:00100000:0.0:1767319679.430551:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876800 00000100:00100000:0.0:1767319679.430555:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876800 00000100:00100000:0.0:1767319679.430581:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 852 00000100:00100000:0.0:1767319679.430591:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982876672:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319679.430624:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 853 00002000:00000024:0.0:1767319679.430628:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319679.430644:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982876800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319679.430676:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319679.430739:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982876800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (656us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319679.430753:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 853 00000100:00100000:2.0:1767319679.430830:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982876800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319679.430901:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982876672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 309us (1803us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319679.430910:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 852 00000100:00100000:3.0:1767319679.431017:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982876672:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319679.432608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319679.432624:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319680.132584:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319680.964496:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319680.964511:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319680.964598:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319680.964602:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319680.964607:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319680.964619:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4924 00000100:00080000:2.0:1767319680.964625:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319680.964634:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319680.964664:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4a80 x1853163982877056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319680.964707:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319680.964710:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319680.964712:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319680.964716:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4924 00000100:00080000:2.0:1767319680.964719:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319680.964723:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319680.964734:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163982877312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319680.964742:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319680.964745:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319680.964747:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319680.964750:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4924 00000100:00080000:2.0:1767319680.964753:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319680.964757:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319680.964774:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163982877440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319680.964812:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319680.964815:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319680.964817:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319680.964821:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4924 00000100:00080000:2.0:1767319680.964823:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319680.964827:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319680.964838:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3100 x1853163982877824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319680.964896:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982877696:0@lo:400:kworker.0 00000100:00100000:2.0:1767319680.965003:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319680.965101:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319680.965107:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982877568:0@lo:400:kworker.0 00000100:00100000:3.0:1767319680.966047:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982876928:0@lo:400:kworker.0 00000100:00100000:3.0:1767319680.966153:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319680.966234:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319680.966243:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982877184:0@lo:400:kworker.0 00000100:00100000:3.0:1767319680.966304:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319680.966350:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982876928 00000100:00100000:3.0:1767319680.966361:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982876928 00000100:00100000:0.0:1767319680.966421:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982877056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966454:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877184 00000100:00100000:0.0:1767319680.966455:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319680.966460:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877184 00000100:00100000:0.0:1767319680.966485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982877312:0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966492:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877056 00000100:00100000:3.0:1767319680.966497:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877056 00000100:00100000:0.0:1767319680.966504:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319680.966513:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877312 00000100:00100000:3.0:1767319680.966518:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877312 00000100:00100000:0.0:1767319680.966519:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982877440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966539:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259 00000100:00100000:0.0:1767319680.966542:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319680.966552:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982876928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319680.966565:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982877824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319680.966581:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319680.966601:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877440 00000100:00100000:0.0:1767319680.966604:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877440 00000100:00100000:0.0:1767319680.966628:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 440 00000100:00100000:3.0:1767319680.966633:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982876928:12345-0@lo:400:kworker.0 Request processed in 83us (478us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319680.966637:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982877440:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259 00010000:00020000:0.0:1767319680.966659:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319680.966661:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877824 00000100:00100000:3.0:1767319680.966665:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877824 00000100:00100000:3.0:1767319680.966674:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260 00000100:00100000:3.0:1767319680.966680:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982877184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319680.966696:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982877440:12345-0@lo:38:kworker.0 Request processed in 60us (153us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319680.966702:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 440 00000100:00100000:3.0:1767319680.966713:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982877184:12345-0@lo:400:kworker.0 Request processed in 33us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319680.966718:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260 00000100:00100000:3.0:1767319680.966731:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1659 00000100:00100000:3.0:1767319680.966734:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982877056:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767319680.966741:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:3.0:1767319680.966743:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319680.966747:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319680.966751:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982877440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966764:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982877056:12345-0@lo:38:kworker.0 Request processed in 28us (310us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319680.966769:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1659 00000100:00080000:0.0:1767319680.966774:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319680.966778:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319680.966781:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1660 00000100:00100000:0.0:1767319680.966784:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982877056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319680.966789:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982877312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319680.966796:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319680.966808:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1661 00000100:00100000:0.0:1767319680.966813:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982877824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319680.966821:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319680.966833:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319680.966836:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319680.966838:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982877312:0@lo:38:kworker.0 00000100:00080000:1.0:1767319680.966915:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319680.966957:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877696 00000100:00100000:1.0:1767319680.966962:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877696 00000100:00100000:1.0:1767319680.967001:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 791 00000100:00100000:1.0:1767319680.967008:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982877696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319680.968494:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982877824:12345-0@lo:38:kworker.0 Request processed in 1671us (1903us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319680.968508:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1661 00000100:00080000:0.0:1767319680.968711:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319680.968717:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319680.968723:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982877824:0@lo:38:kworker.0 00010000:00080000:0.0:1767319680.968834:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5c00 x1853163982877184/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319696 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319680.968848:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982877184:0@lo:400:kworker.0 00010000:00080000:0.0:1767319680.968865:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cf800 x1853163982876928/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319696 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319680.968874:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982876928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319680.969105:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982877696:12345-0@lo:400:kworker.0 Request processed in 2095us (4100us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319680.969249:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 791 00000100:00100000:2.0:1767319680.969444:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319680.969510:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877568 00000100:00100000:1.0:1767319680.969521:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877568 00000100:00100000:2.0:1767319680.969522:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982877696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319680.969710:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 792 00000100:00100000:1.0:1767319680.969722:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982877568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319680.969810:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982877568:12345-0@lo:400:kworker.0 Request processed in 92us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319680.969817:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 792 00000100:00100000:2.0:1767319680.970070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982877568:0@lo:400:kworker.0 00000100:00100000:3.0:1767319680.977140:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982877312:12345-0@lo:38:kworker.0 Request processed in 10335us (10619us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319680.977163:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1660 00010000:00010000:2.0:1767319681.158033:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319682.181064:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319682.956157:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319682.957160:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638656 00000100:00100000:0.0:1767319682.957177:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638656 00000100:00100000:0.0:1767319682.957447:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1261 00000100:00100000:0.0:1767319682.957462:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915638656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319682.957591:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915638656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 133us (1451us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319682.957602:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1261 00000100:00100000:3.0:1767319682.958368:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319682.958607:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638528 00000100:00100000:0.0:1767319682.958616:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638528 00000100:00100000:0.0:1767319682.958651:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1662 00000100:00100000:0.0:1767319682.958659:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319682.958696:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319682.958921:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915638528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 260us (556us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319682.958930:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1662 00000100:00080000:1.0:1767319682.959118:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319683.204702:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319684.228653:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319684.549571:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319684.549605:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982877952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319684.549698:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319684.549836:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982877952 00000100:00100000:2.0:1767319684.549847:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982877952 00000100:00100000:2.0:1767319684.549927:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 854 00000100:00100000:2.0:1767319684.549937:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982877952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319684.549978:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319684.550048:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982877952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 112us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319684.550056:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 854 00000100:00100000:2.0:1767319684.550116:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982877952:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319684.550236:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319684.550444:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319684.550456:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319684.550494:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319684.550622:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878080 00000100:00100000:0.0:1767319684.550629:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878080 00000100:00100000:0.0:1767319684.550716:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 855 00000100:00100000:0.0:1767319684.550724:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319684.550752:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319684.550802:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (307us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319684.550809:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 855 00000100:00100000:3.0:1767319684.550920:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878080:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319684.554500:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319685.252630:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319686.084506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319686.084528:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319686.084613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319686.084618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319686.084622:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319686.084633:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4929 00000100:00080000:2.0:1767319686.084638:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319686.084647:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319686.084682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163982878336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319686.084711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319686.084715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319686.084716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319686.084720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4929 00000100:00080000:2.0:1767319686.084722:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319686.084726:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319686.084736:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543dc00 x1853163982878592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319686.084744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319686.084747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319686.084748:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319686.084751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4929 00000100:00080000:2.0:1767319686.084754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319686.084757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319686.084771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52680 x1853163982878720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319686.084820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319686.084824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319686.084826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319686.084830:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4929 00000100:00080000:2.0:1767319686.084832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319686.084836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319686.084847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51500 x1853163982879104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319686.084906:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878848:0@lo:400:kworker.0 00000100:00100000:2.0:1767319686.085007:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319686.085090:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319686.085097:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878976:0@lo:400:kworker.0 00000100:00100000:2.0:1767319686.085119:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319686.085356:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982878208:0@lo:400:kworker.0 00000100:00100000:1.0:1767319686.085411:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319686.085453:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319686.085460:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982878464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319686.085483:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319686.085526:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878848 00000100:00100000:1.0:1767319686.085537:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878848 00000100:00100000:1.0:1767319686.085617:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878976 00000100:00100000:1.0:1767319686.085621:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878976 00000100:00100000:1.0:1767319686.085646:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 793 00000100:00100000:1.0:1767319686.085657:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319686.085722:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878848:12345-0@lo:400:kworker.0 Request processed in 66us (713us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319686.085731:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 793 00000100:00100000:1.0:1767319686.085751:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 794 00000100:00100000:1.0:1767319686.085757:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319686.085785:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982878976:12345-0@lo:400:kworker.0 Request processed in 28us (665us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319686.085790:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 794 00000100:00080000:1.0:1767319686.085808:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319686.085813:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319686.085855:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319686.085865:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982878848:0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.086111:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878208 00000100:00100000:3.0:1767319686.086117:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878208 00000100:00100000:3.0:1767319686.086163:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878464 00000100:00100000:3.0:1767319686.086177:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878464 00000100:00100000:3.0:1767319686.086204:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1262 00000100:00100000:3.0:1767319686.086212:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982878208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.086255:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982878208:12345-0@lo:400:kworker.0 Request processed in 44us (848us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319686.086261:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1262 00000100:00100000:3.0:1767319686.086309:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263 00000100:00100000:3.0:1767319686.086316:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982878464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319686.086344:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982878464:12345-0@lo:400:kworker.0 Request processed in 29us (860us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319686.086350:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263 00010000:00080000:3.0:1767319686.086403:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f480 x1853163982878464/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319702 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319686.086415:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982878464:0@lo:400:kworker.0 00010000:00080000:3.0:1767319686.086424:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543d500 x1853163982878208/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319702 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319686.086432:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982878208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319686.086609:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982878336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319686.086651:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.086689:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982878592:0@lo:38:kworker.0 00000100:00100000:0.0:1767319686.086714:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.086736:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982878720:0@lo:38:kworker.0 00000100:00100000:0.0:1767319686.086762:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.086785:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982879104:0@lo:38:kworker.0 00000100:00100000:0.0:1767319686.086807:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319686.086837:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878336 00000100:00100000:0.0:1767319686.086842:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878336 00000100:00100000:0.0:1767319686.086858:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878592 00000100:00100000:0.0:1767319686.086860:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878592 00000100:00100000:0.0:1767319686.086868:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879104 00000100:00100000:0.0:1767319686.086871:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879104 00000100:00100000:0.0:1767319686.086880:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1663 00000100:00100000:0.0:1767319686.086885:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982878336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319686.086907:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319686.086946:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982878336:12345-0@lo:38:kworker.0 Request processed in 59us (294us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319686.086952:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1663 00000100:00100000:0.0:1767319686.086965:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1664 00000100:00100000:0.0:1767319686.086968:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982878592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319686.086977:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319686.087001:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982878592:12345-0@lo:38:kworker.0 Request processed in 31us (287us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319686.087007:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1664 00000100:00100000:0.0:1767319686.087017:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665 00000100:00100000:0.0:1767319686.087021:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982879104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319686.087029:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319686.087051:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982879104:12345-0@lo:38:kworker.0 Request processed in 28us (242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319686.087057:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665 00000100:00100000:0.0:1767319686.087075:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982878720 00000100:00100000:0.0:1767319686.087079:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982878720 00000100:00100000:0.0:1767319686.087099:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 441 00000100:00100000:0.0:1767319686.087103:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982878720:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319686.087112:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319686.087137:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982878720:12345-0@lo:38:kworker.0 Request processed in 32us (373us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319686.087144:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 441 00000100:00080000:0.0:1767319686.087189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.087195:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.087199:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982878336:0@lo:38:kworker.0 00000100:00080000:0.0:1767319686.087208:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.087211:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.087215:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982878592:0@lo:38:kworker.0 00000100:00080000:0.0:1767319686.087223:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.087226:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.087229:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982878720:0@lo:38:kworker.0 00000100:00080000:0.0:1767319686.087236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319686.087239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319686.087242:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982879104:0@lo:38:kworker.0 00000100:00080000:1.0:1767319686.087653:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319686.277070:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319687.053369:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915638784 offset 0 length 520: 4 00000100:00100000:3.0:1767319687.053417:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319687.053460:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319687.053511:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639040 00000100:00100000:2.0:1767319687.053530:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639040 00000100:00100000:1.0:1767319687.053633:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915638912 00000100:00100000:1.0:1767319687.053638:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915638912 00000100:00100000:2.0:1767319687.053645:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1264 00000100:00100000:2.0:1767319687.053657:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639040:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319687.053664:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1666 00000100:00100000:1.0:1767319687.053669:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915638912:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319687.053696:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319687.053765:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639040:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 109us (346us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319687.053775:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1264 00000100:00080000:3.0:1767319687.053776:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319687.056927:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915638912:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3235us (3444us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319687.056947:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1666 00010000:00010000:2.0:1767319687.300604:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319688.324770:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319689.348792:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319689.668844:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982879232:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319689.668907:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319689.668979:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319689.668986:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982879360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319689.669007:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319689.669267:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879232 00000100:00100000:0.0:1767319689.669300:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879232 00000100:00100000:0.0:1767319689.669352:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879360 00000100:00100000:0.0:1767319689.669356:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879360 00000100:00100000:0.0:1767319689.669372:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 856 00000100:00100000:0.0:1767319689.669380:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982879232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319689.669409:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319689.669455:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982879232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (547us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319689.669463:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 856 00000100:00100000:0.0:1767319689.669478:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 857 00000100:00100000:0.0:1767319689.669484:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982879360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319689.669498:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319689.669521:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982879360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 37us (513us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319689.669527:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 857 00000100:00080000:0.0:1767319689.669540:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319689.669544:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319689.669572:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982879360:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319689.669586:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982879232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319690.181244:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319665/real 1767319665] req@ffff934604581180 x1853163982872960/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319690 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319690.181295:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604581180 x1853163982872960/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319690 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319690.181314:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.181338:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319690.181342:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982872960:0@lo:250:kworker.0 00000100:00080000:3.0:1767319690.181427:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319690.181432:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319690.181435:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319690.181440:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4914 00000100:00080000:3.0:1767319690.181443:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319690.181450:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319690.181484:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f480 x1853163982879488/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319690.181574:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982879488:0@lo:250:kworker.0 00000400:00000100:2.0:1767319690.181620:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982879488 offset 0 length 520: 4 00000100:00100000:1.0:1767319690.181722:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982879616:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.181760:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319690.181894:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879616 00000100:00100000:2.0:1767319690.181902:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879616 00000100:00100000:2.0:1767319690.181974:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265 00000100:00100000:2.0:1767319690.181982:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982879616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319690.182023:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982879616:12345-0@lo:400:kworker.0 Request processed in 43us (264us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319690.182029:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265 00000100:00080000:1.0:1767319690.182103:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319690.182128:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596300 x1853163982879616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319706 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319690.182138:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982879616:0@lo:400:kworker.0 00000100:00080000:3.0:1767319690.182420:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319690.182425:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319690.182428:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319690.182432:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4934 00000100:00080000:3.0:1767319690.182438:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319690.182442:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319690.182459:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594380 x1853163982879744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319690.182496:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319690.182499:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319690.182501:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319690.182504:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4934 00000100:00080000:3.0:1767319690.182507:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319690.182510:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319690.182524:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597b80 x1853163982880000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319690.182532:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319690.182535:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319690.182537:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319690.182539:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4934 00000100:00080000:3.0:1767319690.182541:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319690.182544:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319690.182555:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596300 x1853163982880128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319690.182564:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319690.182567:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319690.182652:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982879744:0@lo:38:kworker.0 00000100:00100000:0.0:1767319690.182682:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319690.182717:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982880000:0@lo:38:kworker.0 00000100:00100000:0.0:1767319690.182736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319690.182749:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982880128:0@lo:38:kworker.0 00000100:00100000:0.0:1767319690.182771:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319690.182797:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880128 00000100:00100000:0.0:1767319690.182802:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880128 00000100:00100000:0.0:1767319690.182823:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 442 00000100:00100000:0.0:1767319690.182827:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982880128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319690.182844:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319690.182876:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982880128:12345-0@lo:38:kworker.0 Request processed in 47us (103us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319690.182881:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 442 00000100:00080000:0.0:1767319690.182909:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319690.182913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319690.182916:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982880128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319690.183080:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879744 00000100:00100000:2.0:1767319690.183084:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879744 00000100:00100000:2.0:1767319690.183118:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880000 00000100:00100000:2.0:1767319690.183121:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880000 00000100:00100000:2.0:1767319690.183139:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1667 00000100:00100000:2.0:1767319690.183143:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982879744:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319690.183153:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319690.183184:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982879744:12345-0@lo:38:kworker.0 Request processed in 39us (500us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319690.183189:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1667 00000100:00100000:2.0:1767319690.183201:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668 00000100:00100000:2.0:1767319690.183205:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982880000:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319690.183213:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319690.183235:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982880000:12345-0@lo:38:kworker.0 Request processed in 29us (498us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319690.183240:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668 00000100:00080000:2.0:1767319690.183261:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.183265:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.183286:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982879744:0@lo:38:kworker.0 00000100:00080000:2.0:1767319690.183299:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.183302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.183304:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982880000:0@lo:38:kworker.0 00000100:00100000:1.0:1767319690.183380:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319690.183389:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982879872:0@lo:400:kworker.0 00000100:00100000:1.0:1767319690.183418:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319690.183496:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982879872 00000100:00100000:2.0:1767319690.183500:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982879872 00000100:00100000:2.0:1767319690.183541:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1266 00000100:00100000:2.0:1767319690.183547:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982879872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319690.183582:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982879872:12345-0@lo:400:kworker.0 Request processed in 35us (163us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319690.183587:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1266 00000100:00080000:3.0:1767319690.183656:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319690.183662:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4934 00000100:00080000:3.0:1767319690.183665:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319690.183669:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319690.183686:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597b80 x1853163982880256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319690.183804:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982880256:0@lo:38:kworker.0 00000100:00100000:2.0:1767319690.183829:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319690.183854:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880256 00000100:00100000:2.0:1767319690.183858:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880256 00000100:00100000:2.0:1767319690.183880:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1669 00000100:00100000:2.0:1767319690.183887:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982880256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319690.183896:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319690.183920:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982880256:12345-0@lo:38:kworker.0 Request processed in 33us (91us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319690.183926:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1669 00000100:00080000:2.0:1767319690.183943:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319690.183946:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319690.183957:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982880256:0@lo:38:kworker.0 00000100:00080000:1.0:1767319690.184087:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319690.184115:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638595880 x1853163982879872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319706 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319690.184124:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982879872:0@lo:400:kworker.0 00010000:00010000:2.0:1767319690.372596:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319691.396496:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319692.173193:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319692.173310:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639168 00000100:00100000:0.0:1767319692.173320:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639168 00000100:00100000:0.0:1767319692.173369:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1670 00000100:00100000:0.0:1767319692.173377:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915639168:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319692.173399:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319692.173470:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915639168:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 91us (285us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319692.173479:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1670 00000100:00100000:3.0:1767319692.173494:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319692.173506:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639296 00000100:00100000:0.0:1767319692.173512:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639296 00000100:00100000:0.0:1767319692.173554:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1267 00000100:00100000:0.0:1767319692.173563:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915639296:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319692.173613:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915639296:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 52us (122us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319692.173620:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1267 00000100:00080000:3.0:1767319692.173751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319692.420570:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319693.444562:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319694.476452:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319694.789313:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319694.795135:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982880512:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319694.795220:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319694.795262:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982880384:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319694.795314:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319694.795452:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880512 00000100:00100000:3.0:1767319694.795460:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880512 00000100:00100000:3.0:1767319694.795520:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880384 00000100:00100000:3.0:1767319694.795523:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880384 00000100:00100000:3.0:1767319694.795541:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 858 00000100:00100000:3.0:1767319694.795549:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982880512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319694.795580:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319694.795634:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982880512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (412us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319694.795641:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 858 00000100:00100000:3.0:1767319694.795659:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 859 00000100:00100000:3.0:1767319694.795665:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982880384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319694.795681:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319694.795708:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982880384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (392us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319694.795713:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 859 00000100:00100000:3.0:1767319694.795754:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982880512:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319694.795769:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982880384:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319694.795896:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319694.795901:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319695.301373:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.301384:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319695.301448:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.301453:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.301457:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.301467:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.301472:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.301479:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319695.301487:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319695.301502:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982880640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319695.301503:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581180 x1853163982880768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319695.301534:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.301536:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.301538:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.301541:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.301544:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.301547:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319695.301556:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de300 x1853163982881024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319695.301564:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.301567:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.301568:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.301571:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4938 00000100:00100000:3.0:1767319695.301572:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319695.301573:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.301577:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319695.301586:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc700 x1853163982881152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319695.301594:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319695.301597:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319695.301598:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319695.301601:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4938 00000100:00080000:2.0:1767319695.301602:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319695.301605:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319695.301607:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982880896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319695.301614:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982881280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319695.301627:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319695.301652:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880640 00000100:00100000:3.0:1767319695.301652:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880896 00000100:00100000:2.0:1767319695.301658:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880640 00000100:00100000:3.0:1767319695.301658:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880896 00000100:00100000:0.0:1767319695.301707:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982880768:0@lo:38:kworker.0 00000100:00100000:0.0:1767319695.301737:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319695.301755:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982881024:0@lo:38:kworker.0 00000100:00100000:0.0:1767319695.301775:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319695.301792:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982881152:0@lo:38:kworker.0 00000100:00100000:0.0:1767319695.301815:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319695.301830:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982881280:0@lo:38:kworker.0 00000100:00100000:0.0:1767319695.301846:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319695.301872:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982880768 00000100:00100000:0.0:1767319695.301876:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982880768 00000100:00100000:0.0:1767319695.301902:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881024 00000100:00100000:0.0:1767319695.301905:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881024 00000100:00100000:0.0:1767319695.301912:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881280 00000100:00100000:0.0:1767319695.301914:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881280 00000100:00100000:0.0:1767319695.301927:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1671 00000100:00100000:0.0:1767319695.301934:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982880768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319695.301952:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319695.302037:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982880768:12345-0@lo:38:kworker.0 Request processed in 103us (299us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319695.302045:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1671 00000100:00100000:0.0:1767319695.302063:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1672 00000100:00100000:0.0:1767319695.302066:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982881024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319695.302074:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319695.302113:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982881024:12345-0@lo:38:kworker.0 Request processed in 45us (336us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319695.302120:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1672 00000100:00100000:0.0:1767319695.302131:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1673 00000100:00100000:0.0:1767319695.302135:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982881280:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319695.302143:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319695.302165:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982881280:12345-0@lo:38:kworker.0 Request processed in 29us (317us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319695.302171:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1673 00000100:00100000:0.0:1767319695.302193:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881152 00000100:00100000:0.0:1767319695.302197:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881152 00000100:00100000:0.0:1767319695.302216:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 443 00000100:00100000:0.0:1767319695.302220:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982881152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319695.302229:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319695.302267:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982881152:12345-0@lo:38:kworker.0 Request processed in 45us (450us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319695.302319:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 443 00000100:00080000:0.0:1767319695.302356:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319695.302361:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319695.302365:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982880768:0@lo:38:kworker.0 00000100:00080000:0.0:1767319695.302375:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319695.302378:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319695.302380:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982881024:0@lo:38:kworker.0 00000100:00080000:0.0:1767319695.302387:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319695.302390:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319695.302392:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982881152:0@lo:38:kworker.0 00000100:00080000:0.0:1767319695.302399:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319695.302402:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319695.302405:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982881280:0@lo:38:kworker.0 00000100:00100000:3.0:1767319695.302545:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268 00000100:00080000:0.0:1767319695.302593:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319695.303402:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269 00000100:00100000:2.0:1767319695.303414:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982880640:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319695.303498:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982880640:12345-0@lo:400:kworker.0 Request processed in 87us (1925us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319695.303506:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269 00010000:00080000:1.0:1767319695.303923:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5c00 x1853163982880640/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319711 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319695.303938:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982880640:0@lo:400:kworker.0 00000100:00100000:3.0:1767319695.305769:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982880896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319695.305882:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982880896:12345-0@lo:400:kworker.0 Request processed in 3329us (4252us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319695.305893:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268 00010000:00080000:1.0:1767319695.306037:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382df800 x1853163982880896/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319711 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319695.306052:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982880896:0@lo:400:kworker.0 00010000:00010000:2.0:1767319695.492853:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319696.516545:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319697.293647:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319697.293722:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319697.293774:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639424 00000100:00100000:2.0:1767319697.293785:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639424 00000100:00100000:2.0:1767319697.293832:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639552 00000100:00100000:2.0:1767319697.293835:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639552 00000100:00100000:2.0:1767319697.293874:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1270 00000100:00100000:2.0:1767319697.293883:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639552:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319697.294031:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1674 00000100:00100000:0.0:1767319697.294038:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915639424:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319697.294059:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767319697.294142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319697.294791:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639552:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 905us (1064us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319697.294802:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1270 00000100:00100000:0.0:1767319697.302883:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915639424:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 8830us (9273us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319697.302897:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1674 00010000:00010000:1.0:1767319697.543044:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319698.564546:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319699.588623:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319699.908729:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982881408:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319699.908804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319699.909049:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881408 00000100:00100000:0.0:1767319699.909058:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881408 00000100:00100000:0.0:1767319699.909131:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 860 00000100:00100000:0.0:1767319699.909141:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982881408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319699.909170:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319699.909218:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982881408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (413us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319699.909230:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 860 00000100:00100000:0.0:1767319699.909317:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982881408:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319699.909382:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319699.909677:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319699.909688:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982881536:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319699.909726:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319699.909859:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881536 00000100:00100000:2.0:1767319699.909865:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881536 00000100:00100000:2.0:1767319699.909912:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 861 00000100:00100000:2.0:1767319699.909929:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982881536:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319699.909948:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319699.909990:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982881536:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319699.909997:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 861 00000100:00100000:2.0:1767319699.910029:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982881536:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319699.910090:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319700.420631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319700.420643:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319700.420709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319700.420714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319700.420718:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319700.420726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4943 00000100:00080000:3.0:1767319700.420730:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319700.420737:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319700.420759:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624380 x1853163982881792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319700.420781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319700.420784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319700.420785:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319700.420789:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4943 00000100:00080000:3.0:1767319700.420791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319700.420795:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319700.420804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624e00 x1853163982882048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319700.420813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319700.420816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319700.420818:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319700.420821:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4943 00000100:00080000:3.0:1767319700.420823:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319700.420826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319700.420838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982882176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319700.420847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319700.420851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319700.420852:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319700.420856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4943 00000100:00080000:3.0:1767319700.420858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319700.420861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319700.420872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163982882304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319700.420947:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982881664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319700.421002:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982881920:0@lo:400:kworker.0 00000100:00100000:1.0:1767319700.421014:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319700.421725:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881664 00000100:00100000:3.0:1767319700.421734:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881664 00000100:00100000:3.0:1767319700.421788:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271 00000100:00100000:3.0:1767319700.421797:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982881664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319700.421985:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982881792:0@lo:38:kworker.0 00000100:00100000:2.0:1767319700.426479:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319700.426533:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881920 00000100:00100000:1.0:1767319700.426544:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881920 00000100:00080000:2.0:1767319700.426545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319700.426568:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319700.426598:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272 00000100:00100000:1.0:1767319700.426609:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982881920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319700.426681:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982881920:12345-0@lo:400:kworker.0 Request processed in 74us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319700.426681:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982881664:12345-0@lo:400:kworker.0 Request processed in 4883us (5664us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319700.426690:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272 00010000:00080000:2.0:1767319700.426799:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626a00 x1853163982881664/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319716 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319700.426814:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982881664:0@lo:400:kworker.0 00010000:00080000:2.0:1767319700.426842:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624000 x1853163982881920/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319716 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319700.426850:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982881920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319700.426897:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271 00000100:00100000:0.0:1767319700.427371:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319700.427445:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982882048:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.427476:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319700.427562:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982882176:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.427592:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319700.427630:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982882304:0@lo:38:kworker.0 00000100:00100000:0.0:1767319700.427650:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319700.427699:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882176 00000100:00100000:2.0:1767319700.427704:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882176 00000100:00100000:2.0:1767319700.427734:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 444 00000100:00100000:2.0:1767319700.427739:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982882176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319700.427759:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319700.427792:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982882176:12345-0@lo:38:kworker.0 Request processed in 51us (199us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319700.427799:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 444 00000100:00100000:3.0:1767319700.427930:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982881792 00000100:00100000:3.0:1767319700.427934:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982881792 00000100:00100000:3.0:1767319700.428003:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882048 00000100:00100000:3.0:1767319700.428007:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882048 00000100:00080000:0.0:1767319700.428333:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319700.428339:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319700.428343:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982882176:0@lo:38:kworker.0 00000100:00100000:2.0:1767319700.428424:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882304 00000100:00100000:2.0:1767319700.428428:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882304 00000100:00100000:2.0:1767319700.428446:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1675 00000100:00100000:2.0:1767319700.428450:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982881792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319700.428462:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319700.428480:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676 00000100:00100000:1.0:1767319700.428487:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982882048:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319700.428500:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982881792:12345-0@lo:38:kworker.0 Request processed in 47us (1129us total) trans 0 rc -19/-19 00010000:00020000:1.0:1767319700.428502:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319700.428511:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1675 00000100:00100000:2.0:1767319700.428527:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677 00000100:00100000:2.0:1767319700.428532:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982882304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319700.428542:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319700.428544:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982882048:12345-0@lo:38:kworker.0 Request processed in 54us (1065us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319700.428553:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676 00000100:00100000:2.0:1767319700.428573:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982882304:12345-0@lo:38:kworker.0 Request processed in 39us (921us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319700.428579:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677 00000100:00080000:1.0:1767319700.428605:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319700.428611:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319700.428614:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982881792:0@lo:38:kworker.0 00000100:00080000:1.0:1767319700.428623:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319700.428627:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319700.428630:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982882048:0@lo:38:kworker.0 00000100:00080000:1.0:1767319700.428637:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319700.428640:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319700.428643:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982882304:0@lo:38:kworker.0 00010000:00010000:1.0:1767319700.612545:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319701.636576:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319702.429997:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319702.430102:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319702.440584:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639808 00000100:00100000:0.0:1767319702.440600:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639808 00000100:00100000:0.0:1767319702.440750:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639680 00000100:00100000:0.0:1767319702.440754:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639680 00000100:00100000:0.0:1767319702.440777:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1273 00000100:00100000:0.0:1767319702.440787:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319702.440897:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915639808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 111us (10909us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319702.440909:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1273 00000100:00100000:0.0:1767319702.440928:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1678 00000100:00100000:0.0:1767319702.440932:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915639680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319702.440952:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319702.440978:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915639680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (10876us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319702.440983:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1678 00000100:00080000:2.0:1767319702.450008:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319702.663834:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319703.684667:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319704.708506:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319705.028667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982882432:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319705.028739:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319705.028926:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882432 00000100:00100000:3.0:1767319705.028937:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882432 00000100:00100000:3.0:1767319705.029052:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 862 00000100:00100000:3.0:1767319705.029063:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982882432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319705.029106:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319705.029157:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982882432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319705.029164:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 862 00000100:00100000:0.0:1767319705.029266:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982882560:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319705.029562:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319705.029600:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982882432:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319705.029678:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319705.029762:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882560 00000100:00100000:1.0:1767319705.029768:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882560 00000100:00100000:1.0:1767319705.029816:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 863 00000100:00100000:1.0:1767319705.029834:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982882560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319705.029853:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319705.029889:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982882560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 57us (210us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319705.029895:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 863 00000100:00100000:0.0:1767319705.030005:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982882560:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319705.030058:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319705.540853:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.540863:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319705.540925:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.540929:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.540933:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.540940:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.540945:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.540951:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319705.540975:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddc00 x1853163982882816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319705.541080:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982882688:0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.541144:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.541260:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882688 00000100:00100000:0.0:1767319705.541268:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882688 00000100:00100000:0.0:1767319705.541354:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1274 00000100:00100000:0.0:1767319705.541364:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982882688:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.541418:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982882688:12345-0@lo:400:kworker.0 Request processed in 56us (273us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319705.541425:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1274 00010000:00080000:0.0:1767319705.541470:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc380 x1853163982882688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319721 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319705.541484:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982882688:0@lo:400:kworker.0 00000100:00080000:1.0:1767319705.541624:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319705.541842:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.541846:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.541848:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.541853:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.541856:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.541860:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319705.541873:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163982883072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319705.541881:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.541885:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.541887:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.541890:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.541892:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.541896:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319705.541910:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982883200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319705.541941:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319705.541944:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319705.541945:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319705.541949:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4948 00000100:00080000:2.0:1767319705.541951:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319705.541954:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319705.541965:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581180 x1853163982883584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319705.542075:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982882816:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.542106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542133:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982883072:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.542153:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542168:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982883200:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.542190:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542211:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982883584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.542229:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542255:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882816 00000100:00100000:0.0:1767319705.542259:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882816 00000100:00100000:0.0:1767319705.542292:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883072 00000100:00100000:0.0:1767319705.542295:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883072 00000100:00100000:0.0:1767319705.542319:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883584 00000100:00100000:0.0:1767319705.542321:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883584 00000100:00100000:0.0:1767319705.542333:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1679 00000100:00100000:0.0:1767319705.542337:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982882816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319705.542354:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319705.542390:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982882816:12345-0@lo:38:kworker.0 Request processed in 51us (283us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319705.542396:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1679 00000100:00100000:0.0:1767319705.542407:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680 00000100:00100000:0.0:1767319705.542411:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982883072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319705.542419:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319705.542441:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982883072:12345-0@lo:38:kworker.0 Request processed in 28us (287us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319705.542446:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680 00000100:00100000:0.0:1767319705.542455:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1681 00000100:00100000:0.0:1767319705.542458:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982883584:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319705.542466:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319705.542489:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982883584:12345-0@lo:38:kworker.0 Request processed in 29us (259us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319705.542495:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1681 00000100:00080000:0.0:1767319705.542522:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319705.542527:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319705.542530:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982882816:0@lo:38:kworker.0 00000100:00080000:0.0:1767319705.542538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319705.542541:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319705.542544:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982883072:0@lo:38:kworker.0 00000100:00080000:0.0:1767319705.542551:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319705.542554:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319705.542556:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982883584:0@lo:38:kworker.0 00000100:00100000:0.0:1767319705.542594:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982883456:0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.542618:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542637:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319705.542642:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982883328:0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.542660:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319705.542840:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883456 00000100:00100000:0.0:1767319705.542845:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883456 00000100:00100000:0.0:1767319705.542869:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883328 00000100:00100000:0.0:1767319705.542873:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883328 00000100:00100000:0.0:1767319705.542889:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 795 00000100:00100000:0.0:1767319705.542896:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982883456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.542932:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982883456:12345-0@lo:400:kworker.0 Request processed in 37us (312us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319705.542941:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 795 00000100:00100000:0.0:1767319705.542951:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 796 00000100:00100000:0.0:1767319705.542957:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982883328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.543099:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319705.543109:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982882944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.543156:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319705.543182:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982882944 00000100:00100000:1.0:1767319705.543185:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982882944 00000100:00100000:1.0:1767319705.543215:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275 00000100:00100000:1.0:1767319705.543221:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982882944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.543254:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982882944:12345-0@lo:400:kworker.0 Request processed in 33us (97us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319705.543260:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275 00000100:00080000:1.0:1767319705.543299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319705.543331:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc380 x1853163982882944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319721 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319705.543342:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982882944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.543378:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982883328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319705.543388:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982883456:0@lo:400:kworker.0 00000100:00100000:0.0:1767319705.543516:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982883328:12345-0@lo:400:kworker.0 Request processed in 559us (855us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319705.543523:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 796 00000100:00100000:0.0:1767319705.543550:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883200 00000100:00100000:0.0:1767319705.543554:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883200 00000100:00100000:0.0:1767319705.543584:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 445 00000100:00100000:0.0:1767319705.543589:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982883200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319705.543599:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319705.543631:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982883200:12345-0@lo:38:kworker.0 Request processed in 40us (1440us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319705.543637:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 445 00000100:00080000:0.0:1767319705.543658:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319705.543792:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319705.543796:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982883200:0@lo:38:kworker.0 00010000:00010000:2.0:1767319705.739601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319706.756510:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319707.535908:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319707.535992:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319707.537789:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915639936 00000100:00100000:1.0:1767319707.537802:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915639936 00000100:00100000:1.0:1767319707.537848:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640064 00000100:00100000:1.0:1767319707.537852:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640064 00000100:00100000:1.0:1767319707.537906:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1276 00000100:00100000:1.0:1767319707.537916:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915640064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319707.538004:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915640064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (2010us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319707.538012:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1276 00000100:00100000:1.0:1767319707.538032:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1682 00000100:00100000:1.0:1767319707.538036:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915639936:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319707.538056:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319707.538082:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915639936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (2185us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319707.538089:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1682 00000100:00080000:3.0:1767319707.538337:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319707.780739:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319708.805893:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319709.828506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319710.148473:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982883840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319710.148548:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319710.148591:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883840 00000100:00100000:3.0:1767319710.148602:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883840 00000100:00100000:0.0:1767319710.148614:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982883712:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319710.148644:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319710.148706:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319710.148989:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883712 00000100:00100000:3.0:1767319710.148994:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883712 00000100:00100000:3.0:1767319710.149157:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 864 00000100:00100000:3.0:1767319710.149166:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982883840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319710.149197:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319710.149376:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982883840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 210us (827us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319710.149384:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 864 00000100:00100000:3.0:1767319710.149403:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 865 00000100:00100000:3.0:1767319710.149409:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982883712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319710.149466:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319710.149494:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982883712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (849us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319710.149499:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 865 00000100:00100000:3.0:1767319710.149579:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982883840:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319710.149593:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982883712:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319710.151481:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319710.660652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319710.660665:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319710.660724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319710.660728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319710.660732:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319710.660740:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4954 00000100:00080000:3.0:1767319710.660745:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319710.660752:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319710.660775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163982884096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319710.660817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319710.660821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319710.660823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319710.660827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4954 00000100:00080000:3.0:1767319710.660830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319710.660833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319710.660843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982884352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319710.660852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319710.660855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319710.660856:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319710.660860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4954 00000100:00080000:3.0:1767319710.660862:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319710.660865:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319710.660897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163982884480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319710.660918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319710.660922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319710.660923:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319710.660927:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4954 00000100:00080000:3.0:1767319710.660929:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319710.660932:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319710.660942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982884864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319710.661008:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982883968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.661072:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.661131:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319710.661138:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982884736:0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.661166:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319710.661198:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884736 00000100:00100000:2.0:1767319710.661205:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884736 00000100:00100000:2.0:1767319710.661246:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 797 00000100:00100000:2.0:1767319710.661254:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982884736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.661325:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982884736:12345-0@lo:400:kworker.0 Request processed in 72us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319710.661332:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 797 00000100:00100000:2.0:1767319710.661371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982884736:0@lo:400:kworker.0 00000100:00100000:3.0:1767319710.661435:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982883968 00000100:00100000:3.0:1767319710.661439:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982883968 00000100:00100000:3.0:1767319710.661483:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277 00000100:00100000:3.0:1767319710.661489:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982883968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319710.661525:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982883968:12345-0@lo:400:kworker.0 Request processed in 37us (453us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319710.661532:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277 00000100:00100000:1.0:1767319710.661916:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982884224:0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.661946:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.661971:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319710.661977:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982884608:0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.661997:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.662029:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982884096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319710.662052:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.662076:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982884352:0@lo:38:kworker.0 00000100:00100000:1.0:1767319710.662094:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.662109:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982884480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319710.662129:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.662148:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982884864:0@lo:38:kworker.0 00000100:00100000:1.0:1767319710.662166:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319710.662193:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884224 00000100:00100000:1.0:1767319710.662197:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884224 00000100:00100000:1.0:1767319710.662218:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884096 00000100:00100000:1.0:1767319710.662221:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884096 00000100:00100000:1.0:1767319710.662228:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884352 00000100:00100000:1.0:1767319710.662230:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884352 00000100:00100000:1.0:1767319710.662237:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884864 00000100:00100000:1.0:1767319710.662239:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884864 00000100:00100000:1.0:1767319710.662248:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1278 00000100:00100000:1.0:1767319710.662254:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982884224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319710.662308:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982884224:12345-0@lo:400:kworker.0 Request processed in 54us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319710.662316:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1278 00000100:00100000:1.0:1767319710.662330:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683 00000100:00100000:1.0:1767319710.662334:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319710.662352:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319710.662380:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884096:12345-0@lo:38:kworker.0 Request processed in 44us (327us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319710.662386:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683 00000100:00100000:1.0:1767319710.662397:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684 00000100:00100000:1.0:1767319710.662401:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319710.662410:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319710.662432:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884352:12345-0@lo:38:kworker.0 Request processed in 30us (337us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319710.662438:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684 00000100:00100000:1.0:1767319710.662448:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1685 00000100:00100000:1.0:1767319710.662452:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319710.662463:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319710.662484:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982884864:12345-0@lo:38:kworker.0 Request processed in 30us (317us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319710.662490:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1685 00000100:00100000:1.0:1767319710.662511:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884480 00000100:00100000:1.0:1767319710.662515:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884480 00000100:00100000:1.0:1767319710.662531:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 446 00000100:00100000:1.0:1767319710.662536:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982884480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319710.662545:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319710.662568:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982884480:12345-0@lo:38:kworker.0 Request processed in 31us (438us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319710.662573:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 446 00010000:00080000:1.0:1767319710.662607:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fa680 x1853163982884224/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319726 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319710.662618:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982884224:0@lo:400:kworker.0 00000100:00080000:1.0:1767319710.662647:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319710.662652:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319710.662655:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982884096:0@lo:38:kworker.0 00000100:00080000:1.0:1767319710.662663:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319710.662666:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319710.662669:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982884352:0@lo:38:kworker.0 00000100:00080000:1.0:1767319710.662677:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319710.662680:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319710.662683:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982884480:0@lo:38:kworker.0 00000100:00080000:1.0:1767319710.662755:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319710.662759:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319710.662762:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982884864:0@lo:38:kworker.0 00000100:00080000:2.0:1767319710.662901:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319710.662913:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884608 00000100:00100000:2.0:1767319710.662917:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884608 00000100:00100000:2.0:1767319710.662951:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 798 00000100:00100000:2.0:1767319710.662957:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982884608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319710.663045:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982884608:12345-0@lo:400:kworker.0 Request processed in 88us (1046us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319710.663052:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 798 00010000:00080000:2.0:1767319710.663081:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9180 x1853163982883968/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319726 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319710.663092:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982883968:0@lo:400:kworker.0 00000100:00100000:3.0:1767319710.668343:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982884608:0@lo:400:kworker.0 00010000:00010000:2.0:1767319710.852491:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319711.878006:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319712.653835:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319712.653925:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319712.654234:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640192 00000100:00100000:0.0:1767319712.654245:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640192 00000100:00100000:0.0:1767319712.654332:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640320 00000100:00100000:0.0:1767319712.654337:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640320 00000100:00100000:0.0:1767319712.654377:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1279 00000100:00100000:0.0:1767319712.654387:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915640320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319712.654486:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915640320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (559us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319712.654495:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1279 00000100:00100000:0.0:1767319712.654514:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1686 00000100:00100000:0.0:1767319712.654519:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319712.654542:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319712.654569:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (744us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319712.654575:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1686 00000100:00080000:2.0:1767319712.654823:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767319712.658844:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915640448 offset 0 length 520: 4 00010000:00010000:1.0:1767319712.900441:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319713.924542:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319714.951127:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319715.269826:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319715.269853:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982884992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319715.269923:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.270058:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982884992 00000100:00100000:2.0:1767319715.270066:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982884992 00000100:00100000:2.0:1767319715.270125:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 866 00000100:00100000:2.0:1767319715.270134:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982884992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319715.270164:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319715.270205:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982884992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.270212:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 866 00000100:00080000:1.0:1767319715.270312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319715.270351:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982884992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319715.271682:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319715.271699:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982885120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319715.271757:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319715.271860:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885120 00000100:00100000:1.0:1767319715.271867:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885120 00000100:00100000:1.0:1767319715.271929:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 867 00000100:00100000:1.0:1767319715.271937:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982885120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319715.271973:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319715.272013:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982885120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (257us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319715.272020:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 867 00000100:00080000:2.0:1767319715.272287:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319715.272336:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982885120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319715.780399:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319690/real 1767319690] req@ffff93460543f480 x1853163982879488/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319715 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319715.780434:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460543f480 x1853163982879488/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319715 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319715.780455:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319715.780476:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319715.780480:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982879488:0@lo:250:kworker.0 00000100:00080000:0.0:1767319715.780590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.780596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319715.780599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.780603:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4938 00000100:00080000:0.0:1767319715.780607:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.780613:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319715.780673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398e300 x1853163982885248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.780716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.780720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.780722:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.780726:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.780728:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.780732:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319715.780747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163982885504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319715.780751:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982885376:0@lo:400:kworker.0 00000100:00080000:0.0:1767319715.780773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.780777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.780778:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.780781:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.780783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.780786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319715.780797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982885760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.780805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.780808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.780810:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.780812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.780814:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.780817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319715.780820:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319715.780826:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398f100 x1853163982885888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319715.780845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319715.780848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319715.780850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319715.780853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4959 00000100:00080000:0.0:1767319715.780855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319715.780859:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319715.780867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398fb80 x1853163982886272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319715.780876:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319715.780882:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982886144:0@lo:400:kworker.0 00000100:00100000:0.0:1767319715.780887:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982886016:0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.780908:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319715.781051:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982885632:0@lo:400:kworker.0 00000100:00100000:1.0:1767319715.781080:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319715.781356:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886144 00000100:00100000:1.0:1767319715.781364:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886144 00000100:00100000:1.0:1767319715.781409:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 799 00000100:00100000:1.0:1767319715.781419:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982886144:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319715.781467:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982886144:12345-0@lo:400:kworker.0 Request processed in 50us (558us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319715.781475:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 799 00000100:00100000:1.0:1767319715.781501:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982885248:0@lo:250:kworker.0 00000400:00000100:1.0:1767319715.781521:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982885248 offset 0 length 520: 4 00000100:00100000:2.0:1767319715.781522:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885376 00000100:00100000:2.0:1767319715.781525:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885376 00000100:00100000:1.0:1767319715.781540:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982885504:0@lo:38:kworker.0 00000100:00100000:1.0:1767319715.781560:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319715.781734:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982886144:0@lo:400:kworker.0 00000100:00080000:3.0:1767319715.781755:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319715.782040:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982885760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319715.782068:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.782680:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885632 00000100:00100000:2.0:1767319715.782688:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885632 00000100:00100000:2.0:1767319715.782738:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885504 00000100:00100000:2.0:1767319715.782742:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885504 00000100:00100000:2.0:1767319715.782755:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280 00000100:00100000:2.0:1767319715.782763:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982885376:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.782901:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982885376:12345-0@lo:400:kworker.0 Request processed in 138us (2079us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.782910:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280 00000100:00100000:2.0:1767319715.782926:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281 00000100:00100000:2.0:1767319715.782932:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982885632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.783478:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982885632:12345-0@lo:400:kworker.0 Request processed in 543us (2395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.783486:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281 00000100:00100000:2.0:1767319715.783503:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687 00000100:00100000:2.0:1767319715.783507:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982885504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.783525:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.783554:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982885504:12345-0@lo:38:kworker.0 Request processed in 46us (1994us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.783647:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687 00000100:00100000:2.0:1767319715.783697:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319715.783745:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886016 00000100:00100000:2.0:1767319715.783751:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886016 00000100:00100000:2.0:1767319715.783781:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 800 00000100:00100000:2.0:1767319715.783788:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982886016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.783923:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982886016:12345-0@lo:400:kworker.0 Request processed in 135us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319715.783930:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 800 00000100:00080000:2.0:1767319715.783948:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319715.784935:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398c700 x1853163982885632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319731 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:0.0:1767319715.785348:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f480 x1853163982885376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319731 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319715.785362:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982885376:0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.785526:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982885632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319715.785645:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982886016:0@lo:400:kworker.0 00000100:00100000:1.0:1767319715.786240:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982885888:0@lo:38:kworker.0 00000100:00100000:1.0:1767319715.786309:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319715.786339:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982886272:0@lo:38:kworker.0 00000100:00100000:1.0:1767319715.786362:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319715.786394:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319715.786399:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319715.786403:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982885504:0@lo:38:kworker.0 00000100:00100000:0.0:1767319715.786857:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885760 00000100:00100000:0.0:1767319715.786865:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885760 00000100:00100000:0.0:1767319715.786898:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886272 00000100:00100000:0.0:1767319715.786900:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886272 00000100:00100000:0.0:1767319715.786912:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1688 00000100:00100000:0.0:1767319715.786916:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982885760:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319715.787014:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319715.787221:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982885760:12345-0@lo:38:kworker.0 Request processed in 302us (5151us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319715.787229:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1688 00000100:00100000:0.0:1767319715.787246:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689 00000100:00100000:0.0:1767319715.787250:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982886272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319715.787260:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319715.787312:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982886272:12345-0@lo:38:kworker.0 Request processed in 60us (949us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319715.787318:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689 00000100:00100000:2.0:1767319715.787399:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982885888 00000100:00100000:2.0:1767319715.787404:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982885888 00000100:00100000:2.0:1767319715.787434:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 447 00000100:00100000:2.0:1767319715.787437:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982885888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319715.787445:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319715.787467:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982885888:12345-0@lo:38:kworker.0 Request processed in 29us (1160us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319715.787471:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 447 00000100:00080000:1.0:1767319715.787538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319715.787542:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319715.787545:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982885760:0@lo:38:kworker.0 00000100:00080000:1.0:1767319715.787550:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319715.787553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319715.787555:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982885888:0@lo:38:kworker.0 00000100:00080000:1.0:1767319715.787562:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319715.787565:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319715.787567:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982886272:0@lo:38:kworker.0 00010000:00010000:1.0:1767319715.975930:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319716.998005:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319717.779808:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319717.780884:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640704 00000100:00100000:3.0:1767319717.780897:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640704 00000100:00100000:3.0:1767319717.780995:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1282 00000100:00100000:3.0:1767319717.781006:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915640704:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319717.781098:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915640704:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (1303us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319717.781106:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1282 00000100:00100000:3.0:1767319717.781166:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319717.781248:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640576 00000100:00100000:3.0:1767319717.781253:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640576 00000100:00100000:3.0:1767319717.781305:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1690 00000100:00100000:3.0:1767319717.781310:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640576:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319717.781975:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319717.782098:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319717.782148:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640576:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 834us (980us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319717.782158:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1690 00010000:00010000:2.0:1767319718.020507:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319719.044482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319720.069300:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319720.388901:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982886400:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319720.388981:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319720.389031:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982886528:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319720.389072:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.389161:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886400 00000100:00100000:1.0:1767319720.389164:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886528 00000100:00100000:0.0:1767319720.389171:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886400 00000100:00100000:1.0:1767319720.389171:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886528 00000100:00100000:1.0:1767319720.389244:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 868 00000100:00100000:1.0:1767319720.389253:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982886528:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319720.389289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319720.389294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767319720.389308:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319720.389366:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982886528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 114us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319720.389432:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982886528:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319720.390577:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 869 00000100:00100000:0.0:1767319720.390594:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982886400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319720.390622:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319720.390695:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982886400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (1713us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319720.390703:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 869 00000100:00100000:2.0:1767319720.390946:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982886400:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319720.393037:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 868 00000100:00080000:1.0:1767319720.900543:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.900555:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319720.900617:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.900622:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.900626:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.900634:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4964 00000100:00080000:1.0:1767319720.900639:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.900646:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319720.900671:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2d80 x1853163982886784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.900716:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.900719:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.900721:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.900725:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4964 00000100:00080000:1.0:1767319720.900727:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.900731:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319720.900744:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163982887040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.900753:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.900756:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.900767:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.900771:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4964 00000100:00080000:1.0:1767319720.900773:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.900777:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319720.900789:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9c00 x1853163982887168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319720.900809:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319720.900812:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319720.900814:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319720.900817:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4964 00000100:00080000:1.0:1767319720.900820:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319720.900823:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319720.900856:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2d80 x1853163982887552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319720.900895:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982886912:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.901037:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.901074:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319720.901080:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982887296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.901106:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.901143:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886912 00000100:00100000:1.0:1767319720.901151:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886912 00000100:00100000:1.0:1767319720.901206:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283 00000100:00100000:1.0:1767319720.901216:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982886912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.901263:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982886912:12345-0@lo:400:kworker.0 Request processed in 49us (227us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319720.901294:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283 00000100:00100000:2.0:1767319720.901314:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982887424:0@lo:400:kworker.0 00000100:00080000:1.0:1767319720.901320:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319720.901386:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887296 00000100:00100000:0.0:1767319720.901390:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887296 00000100:00100000:0.0:1767319720.901420:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 801 00000100:00100000:0.0:1767319720.901428:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982887296:12345-0@lo:400:kworker.0 00010000:00080000:1.0:1767319720.901469:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f480 x1853163982886912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319736 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319720.901482:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982886912:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.901518:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982886784:0@lo:38:kworker.0 00000100:00100000:3.0:1767319720.901730:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982886656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319720.907342:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319720.907428:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887424 00000100:00100000:2.0:1767319720.907668:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887424 00000100:00100000:2.0:1767319720.907808:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 802 00000100:00100000:2.0:1767319720.907819:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982887424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319720.908070:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982887424:12345-0@lo:400:kworker.0 Request processed in 252us (735us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319720.908079:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 802 00000100:00100000:2.0:1767319720.908149:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982887424:0@lo:400:kworker.0 00000100:00100000:0.0:1767319720.910123:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982887296:12345-0@lo:400:kworker.0 Request processed in 8688us (9008us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319720.910141:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 801 00000100:00100000:2.0:1767319720.910315:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982887296:0@lo:400:kworker.0 00000100:00100000:1.0:1767319720.912138:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.912190:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982887040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319720.912222:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.912398:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982887168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319720.912430:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319720.912452:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982887552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319720.912474:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319720.914802:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887168 00000100:00100000:3.0:1767319720.914816:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887168 00000100:00100000:3.0:1767319720.914867:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 448 00000100:00100000:3.0:1767319720.914874:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982887168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319720.914896:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319720.915057:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982887168:12345-0@lo:38:kworker.0 Request processed in 181us (2624us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319720.915065:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 448 00000100:00080000:1.0:1767319720.915108:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319720.915114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319720.915118:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982887168:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.919036:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886784 00000100:00100000:0.0:1767319720.919052:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886784 00000100:00100000:0.0:1767319720.919131:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887040 00000100:00100000:0.0:1767319720.919135:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887040 00000100:00100000:0.0:1767319720.919159:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887552 00000100:00100000:0.0:1767319720.919162:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887552 00000100:00100000:0.0:1767319720.919177:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1691 00000100:00100000:0.0:1767319720.919186:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982886784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.919209:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.919425:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982886784:12345-0@lo:38:kworker.0 Request processed in 238us (7290us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.919433:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1691 00000100:00100000:0.0:1767319720.919451:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692 00000100:00100000:0.0:1767319720.919455:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982887040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.919468:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.919495:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982887040:12345-0@lo:38:kworker.0 Request processed in 38us (7272us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.919501:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692 00000100:00100000:0.0:1767319720.919512:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1693 00000100:00100000:0.0:1767319720.919516:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982887552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319720.919524:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319720.919560:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982887552:12345-0@lo:38:kworker.0 Request processed in 43us (7085us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319720.919566:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1693 00000100:00100000:0.0:1767319720.919618:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319720.919665:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982886656 00000100:00100000:0.0:1767319720.919753:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982886656 00000100:00100000:0.0:1767319720.919798:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284 00000100:00100000:0.0:1767319720.919804:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982886656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319720.919853:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982886656:12345-0@lo:400:kworker.0 Request processed in 49us (235us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319720.919859:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284 00010000:00080000:0.0:1767319720.919896:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e1180 x1853163982886656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319736 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319720.920038:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319720.920068:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319720.920072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319720.920076:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982886784:0@lo:38:kworker.0 00000100:00080000:3.0:1767319720.920087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319720.920090:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319720.920094:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982887040:0@lo:38:kworker.0 00000100:00080000:3.0:1767319720.920102:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319720.920106:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319720.920109:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982887552:0@lo:38:kworker.0 00000100:00100000:0.0:1767319720.920875:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982886656:0@lo:400:kworker.0 00010000:00010000:2.0:1767319721.100032:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319722.116533:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319722.894959:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319722.895032:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640960 00000100:00100000:2.0:1767319722.895042:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640960 00000100:00100000:2.0:1767319722.895126:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1285 00000100:00100000:2.0:1767319722.895136:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915640960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319722.895213:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915640960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319722.895220:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1285 00000100:00080000:3.0:1767319722.895740:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319722.899716:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319722.899799:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915640832 00000100:00100000:1.0:1767319722.899810:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915640832 00000100:00100000:1.0:1767319722.899858:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1694 00000100:00100000:1.0:1767319722.899867:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319722.899892:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319722.899974:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915640832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 105us (268us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319722.899981:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1694 00010000:00010000:2.0:1767319723.142140:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319724.164522:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319725.188600:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319725.508834:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319725.508858:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982887680:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319725.508938:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319725.509413:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887680 00000100:00100000:2.0:1767319725.509423:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887680 00000100:00100000:2.0:1767319725.509504:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 870 00000100:00100000:2.0:1767319725.509515:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982887680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319725.509550:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767319725.509717:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319725.509843:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982887680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 330us (905us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319725.509852:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 870 00000100:00100000:3.0:1767319725.509939:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982887680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319725.510690:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982887808:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319725.510747:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319725.510997:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887808 00000100:00100000:0.0:1767319725.511005:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887808 00000100:00100000:0.0:1767319725.511078:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 871 00000100:00100000:0.0:1767319725.511087:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982887808:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319725.511248:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:0.0:1767319725.511499:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319725.511610:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982887808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 524us (862us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319725.511618:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 871 00000100:00100000:1.0:1767319725.516745:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982887808:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319726.020767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319726.020777:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319726.020837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319726.020842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319726.020845:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319726.020853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4969 00000100:00080000:0.0:1767319726.020859:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319726.020865:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319726.020889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398df80 x1853163982888064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319726.020932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319726.020935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319726.020937:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319726.020940:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4969 00000100:00080000:0.0:1767319726.020943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319726.020947:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319726.020961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d880 x1853163982888320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319726.020970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319726.020973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319726.020975:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319726.020978:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4969 00000100:00080000:0.0:1767319726.020981:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319726.020985:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319726.021007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982888448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319726.021016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319726.021019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319726.021021:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319726.021024:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4969 00000100:00080000:0.0:1767319726.021026:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319726.021029:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319726.021040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163982888576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319726.021147:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982887936:0@lo:400:kworker.0 00000100:00100000:3.0:1767319726.021211:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319726.021268:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982888192:0@lo:400:kworker.0 00000100:00100000:3.0:1767319726.021315:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319726.021396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982888064:0@lo:38:kworker.0 00000100:00100000:2.0:1767319726.021428:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319726.021465:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982888320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319726.021488:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319726.021504:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982888448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319726.021528:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319726.021563:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982888576:0@lo:38:kworker.0 00000100:00100000:2.0:1767319726.021586:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319726.021616:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982887936 00000100:00100000:2.0:1767319726.021624:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982887936 00000100:00100000:2.0:1767319726.021667:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888192 00000100:00100000:2.0:1767319726.021671:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888192 00000100:00100000:2.0:1767319726.021684:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888064 00000100:00100000:2.0:1767319726.021687:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888064 00000100:00100000:2.0:1767319726.021695:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888320 00000100:00100000:2.0:1767319726.021697:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888320 00000100:00100000:2.0:1767319726.021703:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888576 00000100:00100000:2.0:1767319726.021704:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888576 00000100:00100000:2.0:1767319726.021714:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1286 00000100:00100000:2.0:1767319726.021722:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982887936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319726.021775:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982887936:12345-0@lo:400:kworker.0 Request processed in 53us (562us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319726.021782:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1286 00000100:00100000:2.0:1767319726.021797:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287 00000100:00100000:2.0:1767319726.021803:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982888192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319726.021835:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982888192:12345-0@lo:400:kworker.0 Request processed in 32us (519us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319726.021841:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287 00000100:00100000:2.0:1767319726.021853:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1695 00000100:00100000:2.0:1767319726.021856:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319726.021874:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319726.021906:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888064:12345-0@lo:38:kworker.0 Request processed in 47us (477us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319726.021911:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1695 00000100:00100000:2.0:1767319726.021922:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696 00000100:00100000:2.0:1767319726.021925:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319726.021934:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319726.021955:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888320:12345-0@lo:38:kworker.0 Request processed in 29us (466us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319726.021960:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696 00000100:00100000:2.0:1767319726.021970:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1697 00000100:00100000:2.0:1767319726.021974:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d6c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319726.021983:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319726.022103:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888448 00000100:00100000:3.0:1767319726.022108:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888448 00000100:00100000:3.0:1767319726.022129:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 449 00000100:00100000:3.0:1767319726.022134:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982888448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319726.022144:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319726.022171:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982888448:12345-0@lo:38:kworker.0 Request processed in 36us (643us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319726.022177:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 449 00000100:00080000:3.0:1767319726.022197:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319726.022224:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398d500 x1853163982887936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319742 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319726.022237:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982887936:0@lo:400:kworker.0 00010000:00080000:3.0:1767319726.022298:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398c700 x1853163982888192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319742 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319726.022307:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982888192:0@lo:400:kworker.0 00000100:00080000:3.0:1767319726.022336:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319726.022341:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319726.022344:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982888064:0@lo:38:kworker.0 00000100:00080000:3.0:1767319726.022352:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319726.022355:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319726.022357:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982888320:0@lo:38:kworker.0 00000100:00080000:3.0:1767319726.022364:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319726.022368:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319726.022370:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982888448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319726.022484:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d6c50 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982888576:12345-0@lo:38:kworker.0 Request processed in 507us (895us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319726.022491:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1697 00000100:00080000:3.0:1767319726.023156:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319726.023162:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319726.023165:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982888576:0@lo:38:kworker.0 00010000:00010000:2.0:1767319726.212629:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319727.238587:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319728.012053:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319728.012119:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319728.012318:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641216 00000100:00100000:1.0:1767319728.012329:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641216 00000100:00100000:1.0:1767319728.012503:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641088 00000100:00100000:1.0:1767319728.012507:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641088 00000100:00100000:1.0:1767319728.012531:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1288 00000100:00100000:1.0:1767319728.012542:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915641216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319728.012543:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319728.012588:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1698 00000100:00100000:3.0:1767319728.012597:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915641088:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319728.012616:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319728.012686:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915641088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 88us (564us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319728.012693:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1698 00000100:00100000:1.0:1767319728.013463:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915641216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 919us (1417us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319728.013473:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1288 00010000:00010000:1.0:1767319728.262504:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319729.284599:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319730.308564:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319730.628592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982888704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319730.628664:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319730.628814:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888704 00000100:00100000:0.0:1767319730.628822:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888704 00000100:00100000:0.0:1767319730.628891:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 872 00000100:00100000:0.0:1767319730.628901:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982888704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319730.628929:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319730.628972:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982888704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (307us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319730.628979:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 872 00000100:00080000:1.0:1767319730.629114:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319730.629151:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982888704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319730.634139:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982888832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319730.634209:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319730.634365:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888832 00000100:00100000:0.0:1767319730.634373:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888832 00000100:00100000:0.0:1767319730.634437:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 873 00000100:00100000:0.0:1767319730.634447:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982888832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319730.634484:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319730.634530:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982888832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (319us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319730.634537:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 873 00000100:00080000:1.0:1767319730.634662:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319730.634698:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982888832:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319731.143206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319731.143217:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319731.143303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319731.143308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319731.143312:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319731.143320:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4974 00000100:00080000:3.0:1767319731.143325:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319731.143331:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319731.143362:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1c00 x1853163982889088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319731.143394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319731.143399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319731.143401:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319731.143404:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4974 00000100:00080000:3.0:1767319731.143406:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319731.143409:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319731.143418:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0e00 x1853163982889344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319731.143426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319731.143428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319731.143430:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319731.143433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4974 00000100:00080000:3.0:1767319731.143435:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319731.143438:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319731.143448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0a80 x1853163982889472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319731.143457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319731.143460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319731.143462:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319731.143465:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4974 00000100:00080000:3.0:1767319731.143467:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319731.143469:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319731.143476:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982889088:0@lo:38:kworker.0 00000100:00100000:3.0:1767319731.143477:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163982889600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319731.143498:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982889216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319731.143560:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982888960:0@lo:400:kworker.0 00000100:00100000:3.0:1767319731.143879:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319731.143945:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319731.143972:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982889344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319731.143996:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319731.144018:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982889472:0@lo:38:kworker.0 00000100:00100000:0.0:1767319731.144044:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319731.144073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982889600:0@lo:38:kworker.0 00000100:00100000:0.0:1767319731.144094:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319731.144125:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889472 00000100:00100000:0.0:1767319731.144132:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889472 00000100:00100000:0.0:1767319731.144164:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 450 00000100:00100000:0.0:1767319731.144170:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982889472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319731.144190:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319731.144223:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889216 00000100:00100000:0.0:1767319731.144227:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982889472:12345-0@lo:38:kworker.0 Request processed in 56us (182us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319731.144228:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889216 00000100:00100000:0.0:1767319731.144234:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 450 00000100:00100000:0.0:1767319731.144253:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889088 00000100:00100000:0.0:1767319731.144257:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889088 00000100:00100000:3.0:1767319731.144265:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889344 00000100:00100000:0.0:1767319731.144267:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889600 00000100:00100000:0.0:1767319731.144287:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889600 00000100:00100000:3.0:1767319731.144287:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889344 00000100:00100000:0.0:1767319731.144299:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289 00000100:00100000:0.0:1767319731.144372:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982889216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319731.144416:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982889216:12345-0@lo:400:kworker.0 Request processed in 111us (538us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319731.144488:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319731.144516:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319731.144528:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982888960 00000100:00100000:1.0:1767319731.144531:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982888960 00000100:00100000:3.0:1767319731.150549:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699 00000100:00100000:3.0:1767319731.150565:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319731.150588:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319731.150679:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889088:12345-0@lo:38:kworker.0 Request processed in 117us (6734us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319731.150687:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699 00000100:00100000:3.0:1767319731.150865:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1700 00000100:00100000:3.0:1767319731.150871:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319731.150882:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319731.150910:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889600:12345-0@lo:38:kworker.0 Request processed in 38us (6814us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319731.150916:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1700 00000100:00100000:3.0:1767319731.150929:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701 00000100:00100000:3.0:1767319731.150934:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319731.150943:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319731.150965:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982889344:12345-0@lo:38:kworker.0 Request processed in 30us (6968us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319731.150970:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701 00010000:00080000:3.0:1767319731.151037:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2680 x1853163982889216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319747 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319731.151052:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982889216:0@lo:400:kworker.0 00000100:00080000:3.0:1767319731.151117:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319731.151123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319731.151126:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982889600:0@lo:38:kworker.0 00000100:00080000:3.0:1767319731.151137:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319731.151140:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319731.151143:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982889088:0@lo:38:kworker.0 00000100:00080000:3.0:1767319731.151153:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319731.151156:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319731.151159:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982889344:0@lo:38:kworker.0 00000100:00080000:3.0:1767319731.151169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319731.151173:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319731.151176:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982889472:0@lo:38:kworker.0 00000100:00080000:3.0:1767319731.151212:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319731.157217:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289 00000100:00100000:1.0:1767319731.159115:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1290 00000100:00100000:1.0:1767319731.159142:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982888960:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319731.159250:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982888960:12345-0@lo:400:kworker.0 Request processed in 124us (14761us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319731.159258:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1290 00010000:00080000:3.0:1767319731.159394:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3800 x1853163982888960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319747 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319731.159412:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982888960:0@lo:400:kworker.0 00010000:00010000:1.0:1767319731.340204:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319732.361919:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319733.132116:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319733.132208:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641472 00000100:00100000:1.0:1767319733.132218:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641472 00000100:00080000:3.0:1767319733.132311:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319733.132348:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1291 00000100:00100000:1.0:1767319733.132360:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915641472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319733.132441:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319733.132449:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915641472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (341us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319733.132458:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1291 00000100:00100000:1.0:1767319733.132477:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641344 00000100:00100000:1.0:1767319733.132481:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641344 00000100:00100000:1.0:1767319733.132503:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1702 00000100:00100000:1.0:1767319733.132507:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915641344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319733.132527:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319733.132563:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915641344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (122us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319733.132570:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1702 00010000:00010000:2.0:1767319733.383783:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319734.404522:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319735.428843:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319735.749087:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982889728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319735.749165:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319735.749315:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889728 00000100:00100000:1.0:1767319735.749325:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889728 00000100:00100000:1.0:1767319735.749923:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767319735.749934:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982889728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319735.749964:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319735.750015:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982889728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (850us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319735.750022:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767319735.750076:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982889728:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319735.750628:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982889856:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319735.750664:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319735.750748:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889856 00000100:00100000:0.0:1767319735.750753:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889856 00000100:00080000:2.0:1767319735.750958:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319735.750962:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319735.751221:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 875 00000100:00100000:0.0:1767319735.751228:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982889856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319735.751248:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319735.751345:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982889856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (681us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319735.751359:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 875 00000100:00100000:3.0:1767319735.751807:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982889856:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319736.260510:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319736.260523:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319736.260591:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319736.260595:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319736.260599:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319736.260607:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4979 00000100:00080000:1.0:1767319736.260612:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319736.260620:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319736.260654:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163982890112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319736.260699:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319736.260702:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319736.260704:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319736.260708:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4979 00000100:00080000:1.0:1767319736.260711:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319736.260715:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319736.260726:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0a80 x1853163982890368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319736.260735:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319736.260739:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319736.260741:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319736.260744:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4979 00000100:00080000:1.0:1767319736.260746:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319736.260761:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319736.260774:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163982890496/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319736.260784:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319736.260789:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319736.260790:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319736.260794:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4979 00000100:00080000:1.0:1767319736.260796:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319736.260799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319736.260810:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1180 x1853163982890624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319736.260832:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319736.260849:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982890240:0@lo:400:kworker.0 00000100:00100000:1.0:1767319736.260913:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319736.261057:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982890112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319736.261086:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319736.261121:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982890368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319736.261142:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319736.261159:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982890496:0@lo:38:kworker.0 00000100:00100000:2.0:1767319736.261182:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319736.261201:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982890624:0@lo:38:kworker.0 00000100:00100000:2.0:1767319736.261218:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319736.261306:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890240 00000100:00100000:2.0:1767319736.261315:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890240 00000100:00100000:2.0:1767319736.261361:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890112 00000100:00100000:2.0:1767319736.261364:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890112 00000100:00100000:2.0:1767319736.261372:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890368 00000100:00100000:2.0:1767319736.261375:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890368 00000100:00100000:2.0:1767319736.261382:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890624 00000100:00100000:2.0:1767319736.261384:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890624 00000100:00100000:2.0:1767319736.261393:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292 00000100:00100000:2.0:1767319736.261402:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982890240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319736.261453:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982890240:12345-0@lo:400:kworker.0 Request processed in 53us (541us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319736.261460:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292 00000100:00100000:2.0:1767319736.261477:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1703 00000100:00100000:2.0:1767319736.261481:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319736.261499:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319736.261530:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890112:12345-0@lo:38:kworker.0 Request processed in 47us (443us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319736.261536:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1703 00000100:00100000:2.0:1767319736.261546:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704 00000100:00100000:2.0:1767319736.261549:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319736.261558:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319736.261580:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890368:12345-0@lo:38:kworker.0 Request processed in 28us (437us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319736.261586:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704 00000100:00100000:2.0:1767319736.261596:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1705 00000100:00100000:2.0:1767319736.261600:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319736.261608:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319736.261629:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982890624:12345-0@lo:38:kworker.0 Request processed in 27us (410us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319736.261634:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1705 00000100:00080000:2.0:1767319736.261649:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319736.261674:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319736.261679:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319736.261683:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982890112:0@lo:38:kworker.0 00000100:00080000:2.0:1767319736.261692:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319736.261695:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319736.261698:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982890368:0@lo:38:kworker.0 00000100:00080000:2.0:1767319736.261705:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319736.261708:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319736.261710:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982890624:0@lo:38:kworker.0 00010000:00080000:2.0:1767319736.261746:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f0e00 x1853163982890240/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319752 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319736.261757:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982890240:0@lo:400:kworker.0 00000100:00100000:3.0:1767319736.261848:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890496 00000100:00100000:3.0:1767319736.261852:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890496 00000100:00100000:3.0:1767319736.261885:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 451 00000100:00100000:3.0:1767319736.261890:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982890496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319736.261899:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319736.261932:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982890496:12345-0@lo:38:kworker.0 Request processed in 42us (749us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319736.261937:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 451 00000100:00100000:3.0:1767319736.261965:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982889984:0@lo:400:kworker.0 00000100:00100000:3.0:1767319736.261989:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319736.262014:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982889984 00000100:00100000:3.0:1767319736.262017:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982889984 00000100:00100000:3.0:1767319736.262044:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293 00000100:00100000:3.0:1767319736.262049:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982889984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319736.262079:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982889984:12345-0@lo:400:kworker.0 Request processed in 30us (90us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319736.262085:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293 00000100:00080000:3.0:1767319736.262097:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319736.262113:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1c00 x1853163982889984/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319752 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319736.262121:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982889984:0@lo:400:kworker.0 00000100:00080000:3.0:1767319736.262138:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319736.262141:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319736.262143:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982890496:0@lo:38:kworker.0 00010000:00010000:2.0:1767319736.452536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767319737.229889:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915641600 offset 0 length 520: 4 00000100:00100000:2.0:1767319737.236603:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319737.236749:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641728 00000100:00100000:1.0:1767319737.236760:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641728 00000100:00100000:1.0:1767319737.236808:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1706 00000100:00100000:1.0:1767319737.236816:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915641728:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319737.236839:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319737.236980:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915641728:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 161us (388us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319737.236987:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1706 00000100:00100000:2.0:1767319737.244839:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319737.247584:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641856 00000100:00100000:3.0:1767319737.247602:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641856 00000100:00100000:3.0:1767319737.247704:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1294 00000100:00100000:3.0:1767319737.247716:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915641856:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319737.247772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319737.247801:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915641856:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (2972us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319737.247812:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1294 00010000:00010000:1.0:1767319737.477707:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319738.504535:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319739.527558:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319740.356409:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319715/real 1767319715] req@ffff93463398e300 x1853163982885248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319740 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319740.356449:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463398e300 x1853163982885248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319740 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319740.356466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.356495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767319740.356499:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982885248:0@lo:250:kworker.0 00000100:00080000:3.0:1767319740.360230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319740.360240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319740.360244:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319740.360252:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4964 00000100:00080000:3.0:1767319740.360256:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319740.360263:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319740.360324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0a80 x1853163982890752/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319740.360396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319740.360400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319740.360401:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319740.360405:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4984 00000100:00080000:3.0:1767319740.360407:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319740.360411:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319740.360426:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1c00 x1853163982891008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319740.360448:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319740.360455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319740.360460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319740.360462:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319740.360466:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4984 00000100:00080000:3.0:1767319740.360469:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319740.360472:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982890880:0@lo:400:kworker.0 00000100:00080000:3.0:1767319740.360476:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319740.360489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2680 x1853163982891264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319740.360500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319740.360503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319740.360505:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319740.360508:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4984 00000100:00080000:3.0:1767319740.360511:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319740.360514:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319740.360525:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2d80 x1853163982891392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319740.360545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767319740.360547:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319740.360552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319740.360555:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319740.360559:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4984 00000100:00080000:3.0:1767319740.360562:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319740.360566:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319740.360577:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163982891776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319740.360617:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982891648:0@lo:400:kworker.0 00000100:00100000:3.0:1767319740.360625:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982891136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319740.360646:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319740.360668:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319740.360690:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891648 00000100:00100000:0.0:1767319740.360698:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891648 00000100:00100000:3.0:1767319740.360699:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319740.360707:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982891520:0@lo:400:kworker.0 00000100:00100000:3.0:1767319740.360732:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319740.360756:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891520 00000100:00100000:0.0:1767319740.360761:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891520 00000100:00100000:3.0:1767319740.360763:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982890880 00000100:00100000:3.0:1767319740.360769:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982890880 00000100:00100000:0.0:1767319740.360789:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 803 00000100:00100000:0.0:1767319740.360800:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982891648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319740.360804:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891136 00000100:00100000:3.0:1767319740.360810:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891136 00000100:00100000:3.0:1767319740.360834:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295 00000100:00100000:3.0:1767319740.360842:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982890880:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319740.360859:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982891648:12345-0@lo:400:kworker.0 Request processed in 60us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319740.360869:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 803 00000100:00100000:3.0:1767319740.360882:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982890880:12345-0@lo:400:kworker.0 Request processed in 40us (335us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319740.360890:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 804 00000100:00100000:3.0:1767319740.360891:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295 00000100:00100000:0.0:1767319740.360900:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982891520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319740.360910:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296 00000100:00100000:3.0:1767319740.360918:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982891136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319740.361008:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982890752:0@lo:250:kworker.0 00000100:00080000:1.0:1767319740.361159:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319740.361164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319740.361166:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319740.361222:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982891648:0@lo:400:kworker.0 00010000:00080000:1.0:1767319740.361233:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0e00 x1853163982890880/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319756 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319740.361243:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982890880:0@lo:400:kworker.0 00000100:00100000:1.0:1767319740.361292:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982891520:0@lo:400:kworker.0 00000100:00100000:0.0:1767319740.361401:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982891520:12345-0@lo:400:kworker.0 Request processed in 502us (668us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319740.361408:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 804 00000100:00100000:3.0:1767319740.366789:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982891136:12345-0@lo:400:kworker.0 Request processed in 5860us (6110us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319740.366805:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296 00010000:00080000:1.0:1767319740.367517:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3b80 x1853163982891136/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319756 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319740.367535:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982891136:0@lo:400:kworker.0 00000400:00000100:2.0:1767319740.368045:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982890752 offset 0 length 520: 4 00000100:00100000:2.0:1767319740.368089:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982891008:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368135:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.368163:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982891264:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368216:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.368240:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982891392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368264:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319740.368312:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982891776:0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368335:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319740.368344:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891392 00000100:00100000:0.0:1767319740.368353:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891392 00000100:00100000:2.0:1767319740.368380:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891008 00000100:00100000:2.0:1767319740.368385:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891008 00000100:00100000:0.0:1767319740.368398:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 452 00000100:00100000:0.0:1767319740.368407:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982891392:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368411:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891264 00000100:00100000:2.0:1767319740.368416:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891264 00000100:00100000:2.0:1767319740.368430:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891776 00010000:00020000:0.0:1767319740.368431:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319740.368434:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891776 00000100:00100000:2.0:1767319740.368445:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1707 00000100:00100000:2.0:1767319740.368450:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982891008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319740.368461:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319740.368490:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982891392:12345-0@lo:38:kworker.0 Request processed in 82us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319740.368498:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982891008:12345-0@lo:38:kworker.0 Request processed in 45us (361us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319740.368499:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 452 00000100:00100000:2.0:1767319740.368510:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1707 00000100:00100000:2.0:1767319740.368527:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708 00000100:00100000:2.0:1767319740.368532:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982891264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319740.368542:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319740.368553:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1709 00000100:00100000:0.0:1767319740.368559:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982891776:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319740.368569:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982891264:12345-0@lo:38:kworker.0 Request processed in 34us (351us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767319740.368571:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319740.368581:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708 00000100:00100000:0.0:1767319740.368607:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982891776:12345-0@lo:38:kworker.0 Request processed in 46us (271us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767319740.368614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319740.368615:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1709 00000100:00080000:2.0:1767319740.368624:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.368629:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982891008:0@lo:38:kworker.0 00000100:00080000:2.0:1767319740.368639:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.368643:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.368647:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982891264:0@lo:38:kworker.0 00000100:00080000:2.0:1767319740.368655:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.368659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.368662:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982891392:0@lo:38:kworker.0 00000100:00080000:2.0:1767319740.368669:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319740.368672:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319740.368675:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982891776:0@lo:38:kworker.0 00010000:00010000:1.0:1767319740.556866:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319740.868784:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319740.868810:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982891904:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319740.868882:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319740.869051:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982891904 00000100:00100000:0.0:1767319740.869062:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982891904 00000100:00100000:1.0:1767319740.869075:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982892032:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319740.869110:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319740.872699:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892032 00000100:00100000:0.0:1767319740.872715:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892032 00000100:00100000:0.0:1767319740.872782:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 876 00000100:00100000:0.0:1767319740.872794:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982891904:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319740.872829:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319740.872933:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982891904:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (4051us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319740.873109:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982891904:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319740.873178:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 876 00000100:00100000:0.0:1767319740.873197:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 877 00000100:00100000:0.0:1767319740.873206:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982892032:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319740.873224:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319740.873262:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982892032:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (4152us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319740.873267:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 877 00000100:00100000:1.0:1767319740.873379:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982892032:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319741.573332:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319742.348940:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319742.348981:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319742.349036:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642112 00000100:00100000:1.0:1767319742.349046:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915641984 00000100:00100000:3.0:1767319742.349048:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642112 00000100:00100000:1.0:1767319742.349052:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915641984 00000100:00100000:1.0:1767319742.349099:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1710 00000100:00100000:1.0:1767319742.349107:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915641984:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:3.0:1767319742.349143:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1297 00010000:00020000:1.0:1767319742.349147:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319742.349156:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915642112:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319742.349210:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915641984:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (227us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319742.349217:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915642112:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 62us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319742.349224:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1710 00000100:00100000:3.0:1767319742.349232:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1297 00000100:00080000:2.0:1767319742.349579:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319742.597754:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319743.620474:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319744.644459:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319745.476545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319745.476553:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319745.476623:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982892160:0@lo:400:kworker.0 00000100:00100000:2.0:1767319745.476681:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319745.476733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319745.476736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319745.476739:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319745.476743:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4988 00000100:00080000:3.0:1767319745.476746:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319745.476751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319745.476770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0700 x1853163982892288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319745.476801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319745.476804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767319745.476806:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982892288:0@lo:38:kworker.0 00000100:00080000:3.0:1767319745.476806:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319745.476812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4988 00000100:00080000:3.0:1767319745.476815:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319745.476820:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319745.476829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163982892544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319745.476836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319745.476839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319745.476841:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319745.476843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4988 00000100:00080000:3.0:1767319745.476845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319745.476847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319745.476852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319745.476858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2d80 x1853163982892672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319745.476885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982892544:0@lo:38:kworker.0 00000100:00100000:1.0:1767319745.476906:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.476937:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892160 00000100:00100000:1.0:1767319745.476941:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982892672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319745.476946:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892160 00000100:00100000:1.0:1767319745.476972:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.477009:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892288 00000100:00100000:1.0:1767319745.477010:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982892416:0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.477013:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892288 00000100:00100000:0.0:1767319745.477034:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892544 00000100:00100000:0.0:1767319745.477038:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892544 00000100:00100000:1.0:1767319745.477042:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.477049:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892416 00000100:00100000:0.0:1767319745.477053:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892416 00000100:00100000:1.0:1767319745.477061:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319745.477070:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982892800:0@lo:400:kworker.0 00000100:00100000:0.0:1767319745.477086:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1298 00000100:00100000:0.0:1767319745.477097:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982892160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477108:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319745.477146:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1299 00000100:00100000:1.0:1767319745.477155:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982892416:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477192:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982892416:12345-0@lo:400:kworker.0 Request processed in 39us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319745.477199:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1299 00000100:00100000:1.0:1767319745.477218:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1711 00000100:00100000:1.0:1767319745.477222:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982892288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319745.477239:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319745.477288:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982892288:12345-0@lo:38:kworker.0 Request processed in 47us (420us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319745.477295:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1711 00000100:00100000:1.0:1767319745.477318:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1712 00000100:00100000:1.0:1767319745.477322:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982892544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319745.477330:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319745.477356:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982892544:12345-0@lo:38:kworker.0 Request processed in 32us (448us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319745.477362:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1712 00000100:00100000:1.0:1767319745.477388:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982892928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477410:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319745.477439:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892800 00000100:00100000:1.0:1767319745.477442:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892800 00000100:00100000:1.0:1767319745.477469:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892928 00000100:00100000:1.0:1767319745.477475:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892928 00000100:00080000:3.0:1767319745.477489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319745.477493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319745.477496:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319745.477511:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 805 00000100:00100000:1.0:1767319745.477519:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982892800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477549:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982892800:12345-0@lo:400:kworker.0 Request processed in 31us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319745.477554:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 805 00000100:00100000:1.0:1767319745.477566:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 806 00000100:00100000:1.0:1767319745.477571:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982892928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477600:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982892928:12345-0@lo:400:kworker.0 Request processed in 28us (188us total) trans 0 rc 0/0 00000100:00080000:3.0:1767319745.477602:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4988 00000100:00080000:3.0:1767319745.477608:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319745.477613:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 806 00000100:00080000:3.0:1767319745.477615:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319745.477637:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163982893056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319745.477658:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982892928:0@lo:400:kworker.0 00000100:00100000:1.0:1767319745.477703:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982892800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319745.477711:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982892672 00010000:00080000:1.0:1767319745.477718:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2a00 x1853163982892416/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319761 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319745.477718:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982892672 00000100:00100000:1.0:1767319745.477731:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982892416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319745.477749:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 453 00000100:00100000:1.0:1767319745.477754:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982893056:0@lo:38:kworker.0 00000100:00100000:3.0:1767319745.477757:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982892672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319745.477773:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319745.477778:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982892160:12345-0@lo:400:kworker.0 Request processed in 679us (1090us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319745.477787:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.477791:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1298 00000100:00100000:0.0:1767319745.477810:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893056 00000100:00100000:0.0:1767319745.477814:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893056 00000100:00080000:1.0:1767319745.477822:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319745.477829:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319745.477829:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982892672:12345-0@lo:38:kworker.0 Request processed in 71us (856us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319745.477838:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982892544:0@lo:38:kworker.0 00000100:00100000:0.0:1767319745.477839:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1713 00000100:00100000:3.0:1767319745.477845:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 453 00000100:00100000:0.0:1767319745.477850:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982893056:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767319745.477855:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319745.477861:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767319745.477864:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319745.477868:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982892672:0@lo:38:kworker.0 00000100:00080000:3.0:1767319745.477869:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319745.477876:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319745.477881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319745.477884:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319745.477890:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319745.477895:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982892288:0@lo:38:kworker.0 00000100:00100000:0.0:1767319745.477902:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982893056:12345-0@lo:38:kworker.0 Request processed in 51us (114us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319745.477911:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1713 00000100:00080000:1.0:1767319745.477925:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319745.477930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319745.477934:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982893056:0@lo:38:kworker.0 00010000:00080000:2.0:1767319745.477971:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3480 x1853163982892160/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319761 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319745.477981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982892160:0@lo:400:kworker.0 00010000:00010000:2.0:1767319745.668734:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319745.989742:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982893312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319745.989813:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319745.989918:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893312 00000100:00100000:0.0:1767319745.989929:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893312 00000100:00100000:0.0:1767319745.989988:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 878 00000100:00100000:0.0:1767319745.989997:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982893312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319745.990027:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319745.990090:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982893312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319745.990097:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 878 00000100:00100000:2.0:1767319745.990462:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982893184:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319745.990493:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319745.990545:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893184 00000100:00100000:2.0:1767319745.990549:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893184 00000100:00100000:2.0:1767319745.990572:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767319745.990577:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982893184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319745.990601:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319745.990628:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982893184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 51us (135us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319745.990633:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767319745.990671:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982893184:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319745.990698:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982893312:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319746.692484:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319747.469560:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319747.469700:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642368 00000100:00100000:0.0:1767319747.469712:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642368 00000100:00100000:0.0:1767319747.470072:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1300 00000100:00100000:0.0:1767319747.470084:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915642368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319747.470170:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915642368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (619us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319747.470179:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1300 00000100:00080000:3.0:1767319747.471463:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319747.471655:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319747.471825:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642240 00000100:00100000:0.0:1767319747.473632:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642240 00000100:00100000:0.0:1767319747.473769:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1714 00000100:00100000:0.0:1767319747.473775:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915642240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319747.473794:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319747.473873:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915642240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (2220us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319747.473880:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1714 00010000:00010000:2.0:1767319747.726567:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319748.740490:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319749.764934:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319750.598740:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.598752:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319750.598821:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.598824:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.598827:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.598833:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.598837:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.598843:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319750.598862:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163982893568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.598894:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.598897:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.598899:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.598903:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.598905:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.598909:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319750.598921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ce00 x1853163982893824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.598931:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.598934:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.598936:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.598940:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.598944:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767319750.598945:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982893440:0@lo:400:kworker.0 00000100:00080000:1.0:1767319750.598950:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319750.598966:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f800 x1853163982893952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319750.598999:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319750.599002:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319750.599004:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319750.599007:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4994 00000100:00080000:1.0:1767319750.599009:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319750.599013:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319750.599022:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c380 x1853163982894336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319750.599052:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982893696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.599113:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.599151:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319750.599157:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982894080:0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.599181:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319750.599213:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894080 00000100:00100000:1.0:1767319750.599220:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894080 00000100:00100000:1.0:1767319750.599260:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 807 00000100:00100000:1.0:1767319750.599265:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982894080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319750.599325:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982894080:12345-0@lo:400:kworker.0 Request processed in 59us (143us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319750.599332:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 807 00000100:00080000:1.0:1767319750.599365:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319750.599408:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982894080:0@lo:400:kworker.0 00000100:00100000:0.0:1767319750.599471:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982894208:0@lo:400:kworker.0 00000100:00100000:0.0:1767319750.599494:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319750.599530:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893696 00000100:00100000:0.0:1767319750.599534:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893696 00000100:00100000:0.0:1767319750.599561:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1301 00000100:00100000:0.0:1767319750.599566:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982893696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319750.599591:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982893696:12345-0@lo:400:kworker.0 Request processed in 25us (479us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319750.599595:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1301 00000100:00100000:3.0:1767319750.599621:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.599885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982893568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.599915:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319750.603841:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894208 00000100:00100000:3.0:1767319750.603858:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894208 00000100:00100000:3.0:1767319750.603929:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 808 00000100:00100000:3.0:1767319750.603940:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982894208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319750.604049:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982894208:12345-0@lo:400:kworker.0 Request processed in 111us (4552us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319750.604061:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 808 00000100:00080000:3.0:1767319750.604094:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319750.604098:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319750.604148:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543c700 x1853163982893696/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319766 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319750.604163:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982893696:0@lo:400:kworker.0 00000100:00100000:3.0:1767319750.604207:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982894208:0@lo:400:kworker.0 00000100:00100000:2.0:1767319750.609592:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982893824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319750.609597:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893440 00000100:00100000:2.0:1767319750.609660:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.609679:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982893952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.609698:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.609716:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893952 00000100:00100000:2.0:1767319750.609722:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893952 00000100:00100000:2.0:1767319750.609751:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 454 00000100:00100000:2.0:1767319750.609756:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982893952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319750.609771:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.609799:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982893952:12345-0@lo:38:kworker.0 Request processed in 43us (99us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.609803:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 454 00000100:00080000:2.0:1767319750.609843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.609846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.609849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982893952:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.609854:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982894336:0@lo:38:kworker.0 00000100:00100000:2.0:1767319750.609868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319750.609905:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893568 00000100:00100000:2.0:1767319750.609908:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893568 00000100:00100000:2.0:1767319750.609919:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982893824 00000100:00100000:2.0:1767319750.609921:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893824 00000100:00100000:2.0:1767319750.609928:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894336 00000100:00100000:2.0:1767319750.609930:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894336 00000100:00100000:2.0:1767319750.609939:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1715 00000100:00100000:2.0:1767319750.609944:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982893568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319750.609954:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.610000:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982893568:12345-0@lo:38:kworker.0 Request processed in 55us (10085us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.610006:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1715 00000100:00100000:2.0:1767319750.610018:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716 00000100:00100000:2.0:1767319750.610022:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982893824:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319750.610032:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.610055:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982893824:12345-0@lo:38:kworker.0 Request processed in 31us (396us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.610183:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716 00000100:00100000:2.0:1767319750.610197:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1717 00000100:00100000:2.0:1767319750.610201:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982894336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319750.610210:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319750.610234:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982894336:12345-0@lo:38:kworker.0 Request processed in 32us (365us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319750.610240:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1717 00000100:00080000:2.0:1767319750.610263:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.610266:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.610283:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982893568:0@lo:38:kworker.0 00000100:00080000:2.0:1767319750.610290:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.610293:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.610295:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982893824:0@lo:38:kworker.0 00000100:00080000:2.0:1767319750.610299:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319750.610301:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319750.610302:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982894336:0@lo:38:kworker.0 00000100:00100000:0.0:1767319750.610399:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982893440 00000100:00100000:0.0:1767319750.610459:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1302 00000100:00100000:0.0:1767319750.610476:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982893440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319750.610521:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982893440:12345-0@lo:400:kworker.0 Request processed in 46us (10901us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319750.610527:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1302 00010000:00080000:0.0:1767319750.610556:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f0a80 x1853163982893440/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319766 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319750.610569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982893440:0@lo:400:kworker.0 00000100:00080000:2.0:1767319750.610930:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319750.798869:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319751.108859:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319751.108880:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982894464:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319751.108947:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319751.109013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982894592:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319751.109044:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894464 00000100:00100000:0.0:1767319751.109055:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894464 00000100:00100000:0.0:1767319751.109103:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 880 00000100:00100000:0.0:1767319751.109113:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982894464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319751.109144:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319751.111307:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319751.111805:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894592 00000100:00100000:2.0:1767319751.111817:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894592 00000100:00100000:2.0:1767319751.111875:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 881 00000100:00100000:2.0:1767319751.111886:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982894592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319751.111921:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319751.111989:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982894592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (690us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319751.111998:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 881 00000100:00100000:1.0:1767319751.112039:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982894592:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319751.112363:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982894464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3250us (3414us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319751.112372:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 880 00000100:00100000:2.0:1767319751.114171:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982894464:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319751.814312:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319752.588399:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319752.591410:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642624 00000100:00100000:3.0:1767319752.592017:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642624 00000100:00100000:3.0:1767319752.592115:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1303 00000100:00100000:3.0:1767319752.592125:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915642624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319752.592443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319752.597293:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915642624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 5144us (8879us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319752.597308:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1303 00000100:00100000:2.0:1767319752.602409:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319752.602542:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642496 00000100:00100000:3.0:1767319752.602563:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642496 00000100:00100000:3.0:1767319752.602637:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1718 00000100:00100000:3.0:1767319752.602649:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915642496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319752.602678:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319752.603084:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915642496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 433us (695us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319752.603105:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1718 00010000:00010000:1.0:1767319752.836525:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319753.860553:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319754.884503:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319755.716472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.716482:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319755.716547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.716551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.716554:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.716563:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.716568:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.716574:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319755.716596:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0700 x1853163982894848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.716637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.716640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.716642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.716645:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.716647:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.716651:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319755.716661:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163982895104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.716670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.716673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.716674:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.716678:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.716680:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.716684:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319755.716693:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982895232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319755.716715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319755.716718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319755.716720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319755.716723:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 4999 00000100:00080000:3.0:1767319755.716725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319755.716729:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319755.716737:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1880 x1853163982895616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319755.716804:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982895488:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.716873:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.716966:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319755.716973:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982894720:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.717005:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.717035:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982895360:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.717056:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.717072:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319755.717076:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982894976:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.717096:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319755.717125:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894720 00000100:00100000:1.0:1767319755.717138:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894720 00000100:00100000:1.0:1767319755.717194:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894976 00000100:00100000:1.0:1767319755.717198:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894976 00000100:00100000:1.0:1767319755.717224:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1304 00000100:00100000:1.0:1767319755.717232:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982894720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.717307:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982894720:12345-0@lo:400:kworker.0 Request processed in 76us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319755.717315:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1304 00000100:00100000:1.0:1767319755.717332:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1305 00000100:00100000:1.0:1767319755.717338:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982894976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.717370:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982894976:12345-0@lo:400:kworker.0 Request processed in 32us (272us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319755.717376:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1305 00010000:00080000:1.0:1767319755.717408:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2d80 x1853163982894976/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319771 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319755.717420:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982894976:0@lo:400:kworker.0 00010000:00080000:1.0:1767319755.717443:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3b80 x1853163982894720/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319771 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319755.717451:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982894720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319755.717520:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895488 00000100:00100000:3.0:1767319755.717529:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895488 00000100:00100000:3.0:1767319755.717561:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895360 00000100:00100000:3.0:1767319755.717564:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895360 00000100:00100000:3.0:1767319755.717584:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 809 00000100:00100000:3.0:1767319755.717590:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982895488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319755.717624:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982895488:12345-0@lo:400:kworker.0 Request processed in 35us (753us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319755.717630:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 809 00000100:00100000:3.0:1767319755.717641:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 810 00000100:00100000:3.0:1767319755.717646:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982895360:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319755.717675:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982895360:12345-0@lo:400:kworker.0 Request processed in 29us (618us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319755.717680:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 810 00000100:00100000:3.0:1767319755.717712:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982895488:0@lo:400:kworker.0 00000100:00100000:1.0:1767319755.721083:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982895360:0@lo:400:kworker.0 00000100:00080000:2.0:1767319755.721205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319755.721211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319755.721214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319755.721344:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982894848:0@lo:38:kworker.0 00000100:00100000:0.0:1767319755.721413:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319755.721452:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982895104:0@lo:38:kworker.0 00000100:00100000:0.0:1767319755.721476:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319755.721504:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982895232:0@lo:38:kworker.0 00000100:00100000:0.0:1767319755.721529:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319755.721552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982895616:0@lo:38:kworker.0 00000100:00100000:0.0:1767319755.721573:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319755.721609:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982894848 00000100:00100000:0.0:1767319755.721616:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982894848 00000100:00100000:0.0:1767319755.721647:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895104 00000100:00100000:0.0:1767319755.721650:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895104 00000100:00100000:0.0:1767319755.721662:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895616 00000100:00100000:0.0:1767319755.721664:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895616 00000100:00100000:0.0:1767319755.721676:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1719 00000100:00100000:0.0:1767319755.721684:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982894848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319755.721707:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319755.721748:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982894848:12345-0@lo:38:kworker.0 Request processed in 64us (335us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319755.721755:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1719 00000100:00100000:0.0:1767319755.721770:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720 00000100:00100000:0.0:1767319755.721774:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982895104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319755.721783:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319755.721803:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982895104:12345-0@lo:38:kworker.0 Request processed in 28us (327us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319755.721809:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720 00000100:00100000:0.0:1767319755.721817:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1721 00000100:00100000:0.0:1767319755.721821:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982895616:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319755.721829:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319755.721845:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982895616:12345-0@lo:38:kworker.0 Request processed in 23us (271us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319755.721850:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1721 00000100:00080000:0.0:1767319755.721875:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319755.721880:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319755.721896:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982894848:0@lo:38:kworker.0 00000100:00080000:0.0:1767319755.721906:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319755.721909:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319755.721912:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982895104:0@lo:38:kworker.0 00000100:00080000:0.0:1767319755.721918:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319755.721921:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319755.721923:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982895616:0@lo:38:kworker.0 00000100:00100000:2.0:1767319755.722004:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895232 00000100:00100000:2.0:1767319755.722008:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895232 00000100:00100000:2.0:1767319755.722043:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 455 00000100:00100000:2.0:1767319755.722049:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982895232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319755.722060:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319755.722095:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982895232:12345-0@lo:38:kworker.0 Request processed in 46us (565us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319755.722101:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 455 00000100:00080000:2.0:1767319755.722127:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319755.722131:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319755.722134:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982895232:0@lo:38:kworker.0 00010000:00010000:1.0:1767319755.908540:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319756.228567:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982895744:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319756.228639:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319756.228800:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895744 00000100:00100000:3.0:1767319756.228808:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895744 00000100:00100000:3.0:1767319756.228862:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 882 00000100:00100000:3.0:1767319756.228870:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982895744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319756.228909:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319756.228962:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982895744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319756.228970:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 882 00000100:00100000:2.0:1767319756.233981:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982895744:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319756.234261:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982895872:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319756.234359:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319756.234476:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982895872 00000100:00100000:0.0:1767319756.234483:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982895872 00000100:00100000:0.0:1767319756.234530:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 883 00000100:00100000:0.0:1767319756.234540:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982895872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319756.234566:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319756.234624:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982895872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319756.234632:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 883 00000100:00100000:2.0:1767319756.234836:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982895872:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319756.933036:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319757.722150:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319757.722205:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319757.722378:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642752 00000100:00100000:2.0:1767319757.722390:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642752 00000100:00100000:2.0:1767319757.722447:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915642880 00000100:00100000:2.0:1767319757.722451:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915642880 00000100:00100000:2.0:1767319757.722508:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1306 00000100:00100000:2.0:1767319757.722518:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915642880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319757.722565:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1722 00000100:00100000:1.0:1767319757.722574:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915642752:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767319757.722596:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915642880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (388us total) trans 0 rc 0/0 00010000:00020000:1.0:1767319757.722597:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319757.722609:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1306 00000100:00100000:1.0:1767319757.722641:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915642752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 66us (514us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319757.722650:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1722 00000100:00080000:0.0:1767319757.722883:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319757.956529:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319758.980955:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319760.004428:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319760.836461:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319760.836475:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319760.836544:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319760.836548:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319760.836552:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319760.836560:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5004 00000100:00080000:1.0:1767319760.836565:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319760.836571:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319760.836592:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f480 x1853163982896128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319760.836635:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319760.836638:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319760.836640:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319760.836654:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5004 00000100:00080000:1.0:1767319760.836656:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319760.836660:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319760.836671:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ce00 x1853163982896384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319760.836679:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319760.836681:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319760.836683:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319760.836686:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5004 00000100:00080000:1.0:1767319760.836687:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319760.836690:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319760.836698:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f800 x1853163982896512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319760.836714:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319760.836717:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319760.836718:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319760.836721:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5004 00000100:00080000:1.0:1767319760.836723:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319760.836725:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319760.836737:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0e00 x1853163982896896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319760.836768:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982896256:0@lo:400:kworker.0 00000100:00100000:3.0:1767319760.836836:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982896640:0@lo:400:kworker.0 00000100:00100000:1.0:1767319760.836875:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319760.836875:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319760.836960:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982896000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319760.836994:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319760.837032:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982896128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319760.837032:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319760.837043:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982896768:0@lo:400:kworker.0 00000100:00100000:0.0:1767319760.837073:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319760.837098:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982896384:0@lo:38:kworker.0 00000100:00100000:0.0:1767319760.837117:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319760.840883:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982896512:0@lo:38:kworker.0 00000100:00100000:0.0:1767319760.841095:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319760.841126:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982896896:0@lo:38:kworker.0 00000100:00100000:0.0:1767319760.841157:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319760.841322:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896256 00000100:00100000:2.0:1767319760.841334:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896256 00000100:00100000:2.0:1767319760.841405:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896000 00000100:00100000:2.0:1767319760.841408:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896000 00000100:00100000:2.0:1767319760.841423:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896128 00000100:00100000:2.0:1767319760.841426:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896128 00000100:00100000:2.0:1767319760.841434:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896384 00000100:00100000:2.0:1767319760.841436:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896384 00000100:00100000:2.0:1767319760.841447:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1307 00000100:00100000:2.0:1767319760.841456:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982896256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319760.841528:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982896256:12345-0@lo:400:kworker.0 Request processed in 73us (4653us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319760.841535:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1307 00000100:00100000:2.0:1767319760.841552:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896896 00000100:00100000:2.0:1767319760.841555:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896896 00000100:00100000:2.0:1767319760.841566:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1308 00000100:00100000:2.0:1767319760.841573:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982896000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319760.841594:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896512 00000100:00100000:0.0:1767319760.841600:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896512 00000100:00100000:2.0:1767319760.841608:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982896000:12345-0@lo:400:kworker.0 Request processed in 36us (4614us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319760.841616:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1308 00000100:00100000:0.0:1767319760.841633:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 456 00000100:00100000:2.0:1767319760.841633:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1723 00000100:00100000:2.0:1767319760.841645:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319760.841664:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319760.841694:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896128:12345-0@lo:38:kworker.0 Request processed in 51us (4622us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319760.841700:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1723 00000100:00100000:2.0:1767319760.841709:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1724 00000100:00100000:2.0:1767319760.841713:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319760.841721:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319760.841725:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982896512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319760.841739:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319760.841746:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896384:12345-0@lo:38:kworker.0 Request processed in 30us (4626us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319760.841752:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1724 00000100:00100000:2.0:1767319760.841763:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1725 00000100:00100000:2.0:1767319760.841768:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319760.841780:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319760.841781:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982896512:12345-0@lo:38:kworker.0 Request processed in 136us (684us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319760.841820:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982896896:12345-0@lo:38:kworker.0 Request processed in 51us (662us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319760.841826:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1725 00000100:00080000:2.0:1767319760.841843:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319760.841875:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 456 00000100:00080000:0.0:1767319760.842070:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319760.842076:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319760.842079:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982896128:0@lo:38:kworker.0 00000100:00080000:0.0:1767319760.842088:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319760.842092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319760.842094:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982896384:0@lo:38:kworker.0 00000100:00080000:0.0:1767319760.842100:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319760.842103:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319760.842106:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982896512:0@lo:38:kworker.0 00000100:00080000:0.0:1767319760.842112:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319760.842115:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319760.842118:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982896896:0@lo:38:kworker.0 00000100:00100000:3.0:1767319760.843458:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896640 00000100:00100000:3.0:1767319760.843469:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896640 00000100:00100000:3.0:1767319760.843527:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 811 00000100:00100000:3.0:1767319760.843535:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982896640:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319760.843601:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982896640:12345-0@lo:400:kworker.0 Request processed in 67us (6725us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319760.843607:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 811 00000100:00080000:3.0:1767319760.843634:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319760.843671:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982896640:0@lo:400:kworker.0 00010000:00080000:1.0:1767319760.846480:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543c380 x1853163982896256/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319776 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319760.846502:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982896256:0@lo:400:kworker.0 00000100:00100000:2.0:1767319760.848135:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767319760.848210:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543dc00 x1853163982896000/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319776 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319760.848226:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982896000:0@lo:400:kworker.0 00000100:00100000:3.0:1767319760.850642:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982896768 00000100:00100000:3.0:1767319760.850669:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982896768 00000100:00100000:3.0:1767319760.850770:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 812 00000100:00100000:3.0:1767319760.850780:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982896768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319760.850890:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982896768:12345-0@lo:400:kworker.0 Request processed in 111us (2762us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319760.850901:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 812 00000100:00080000:2.0:1767319760.853914:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319760.853980:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982896768:0@lo:400:kworker.0 00010000:00010000:2.0:1767319761.028550:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319761.349088:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982897024:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319761.349162:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319761.349177:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982897152:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319761.349215:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319761.349265:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897024 00000100:00100000:2.0:1767319761.349292:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897024 00000100:00100000:2.0:1767319761.349359:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897152 00000100:00100000:2.0:1767319761.349363:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897152 00000100:00100000:2.0:1767319761.349380:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 884 00000100:00100000:2.0:1767319761.349389:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982897024:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319761.349421:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319761.349467:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982897024:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (304us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319761.349474:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 884 00000100:00100000:0.0:1767319761.349537:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 885 00000100:00100000:0.0:1767319761.349549:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982897152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319761.349566:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319761.349651:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982897152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (434us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319761.349658:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 885 00000100:00100000:3.0:1767319761.349720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982897024:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319761.349734:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982897152:0@lo:13:osp-pre-1-1.0 00000400:00000100:0.0:1767319761.804527:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915643008 offset 0 length 520: 4 00000100:00100000:0.0:1767319761.804602:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319761.804931:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643136 00000100:00100000:3.0:1767319761.804942:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643136 00000100:00100000:3.0:1767319761.804993:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1726 00000100:00100000:3.0:1767319761.805001:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915643136:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319761.805026:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319761.805100:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915643136:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 99us (498us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319761.805108:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1726 00000100:00100000:2.0:1767319761.815577:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319761.815838:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643264 00000100:00100000:0.0:1767319761.815851:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643264 00000100:00100000:0.0:1767319761.815933:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1309 00000100:00100000:0.0:1767319761.815943:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915643264:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319761.816036:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915643264:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (470us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319761.816044:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1309 00000100:00080000:2.0:1767319761.819920:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319762.052456:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319763.079073:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319764.100490:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319765.124509:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319765.956616:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319740/real 1767319740] req@ffff934609af0a80 x1853163982890752/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319765.956648:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609af0a80 x1853163982890752/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319765.956662:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319765.956670:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319765.956674:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982890752:0@lo:250:kworker.0 00000100:00080000:1.0:1767319765.956906:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319765.956914:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319765.956918:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319765.956925:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 4988 00000100:00080000:1.0:1767319765.956930:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319765.956939:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319765.956994:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0380 x1853163982897280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319765.957051:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319765.957054:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319765.957055:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319765.957059:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5009 00000100:00080000:1.0:1767319765.957061:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319765.957064:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319765.957081:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982897536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319765.957115:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319765.957120:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319765.957122:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767319765.957123:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319765.957134:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5009 00000100:00080000:1.0:1767319765.957137:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319765.957142:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319765.957147:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982897408:0@lo:400:kworker.0 00000100:00100000:1.0:1767319765.957156:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982897792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319765.957164:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319765.957167:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319765.957169:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319765.957173:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5009 00000100:00080000:1.0:1767319765.957175:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319765.957177:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319765.957189:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1500 x1853163982897920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319765.957210:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319765.957212:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319765.957214:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319765.957217:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5009 00000100:00080000:1.0:1767319765.957220:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319765.957225:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319765.957234:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.957239:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163982898304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319765.957292:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982897664:0@lo:400:kworker.0 00000100:00100000:2.0:1767319765.957330:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982898176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319765.957352:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319765.957432:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897408 00000100:00100000:0.0:1767319765.957440:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897408 00000100:00100000:0.0:1767319765.957488:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897664 00000100:00100000:0.0:1767319765.957490:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897664 00000100:00100000:0.0:1767319765.957509:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1310 00000100:00100000:0.0:1767319765.957515:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982897408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319765.957577:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982898048:0@lo:400:kworker.0 00000100:00100000:2.0:1767319765.957674:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319765.957718:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898176 00000100:00100000:2.0:1767319765.957721:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898176 00000100:00100000:2.0:1767319765.957747:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 813 00000100:00100000:2.0:1767319765.957753:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982898176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319765.957786:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982898176:12345-0@lo:400:kworker.0 Request processed in 32us (111us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319765.957792:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 813 00000100:00100000:2.0:1767319765.957812:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1311 00000100:00080000:1.0:1767319765.958197:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319765.958205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319765.958220:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982897280:0@lo:250:kworker.0 00000400:00000100:1.0:1767319765.958262:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982897280 offset 0 length 520: 4 00000100:00100000:1.0:1767319765.958307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982897536:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.958326:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.958341:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982897792:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.958352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.958358:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982897920:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.958426:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.958442:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982898304:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.958459:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.958640:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897536 00000100:00100000:1.0:1767319765.958647:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897536 00000100:00100000:1.0:1767319765.958673:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897792 00000100:00100000:1.0:1767319765.958676:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897792 00000100:00100000:1.0:1767319765.958683:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898304 00000100:00100000:1.0:1767319765.958685:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898304 00000100:00100000:1.0:1767319765.958708:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1727 00000100:00100000:1.0:1767319765.958714:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982897536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319765.958739:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319765.959121:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982897536:12345-0@lo:38:kworker.0 Request processed in 404us (791us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319765.959130:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1727 00000100:00100000:1.0:1767319765.959152:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728 00000100:00100000:1.0:1767319765.959156:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982897792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319765.959166:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319765.959196:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982897792:12345-0@lo:38:kworker.0 Request processed in 38us (841us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319765.959202:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728 00000100:00100000:1.0:1767319765.959214:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1729 00000100:00100000:1.0:1767319765.959218:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982898304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319765.959225:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319765.959243:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982898304:12345-0@lo:38:kworker.0 Request processed in 24us (783us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319765.959248:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1729 00000100:00080000:1.0:1767319765.959327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319765.959332:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319765.959335:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982897536:0@lo:38:kworker.0 00000100:00080000:1.0:1767319765.959343:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319765.959346:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319765.959348:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982897792:0@lo:38:kworker.0 00000100:00080000:1.0:1767319765.959353:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319765.959355:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319765.959357:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982898304:0@lo:38:kworker.0 00000100:00100000:1.0:1767319765.959462:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982897920 00000100:00100000:1.0:1767319765.959466:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982897920 00000100:00100000:2.0:1767319765.959582:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982897664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319765.959628:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 457 00000100:00100000:1.0:1767319765.959634:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982897920:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319765.959643:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319765.959717:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982897920:12345-0@lo:38:kworker.0 Request processed in 73us (1280us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319765.959718:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982897664:12345-0@lo:400:kworker.0 Request processed in 1888us (2354us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319765.959745:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 457 00000100:00100000:2.0:1767319765.959757:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1311 00010000:00080000:2.0:1767319765.959998:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1c00 x1853163982897664/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319781 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319765.960025:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982897664:0@lo:400:kworker.0 00000100:00080000:1.0:1767319765.960236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319765.960241:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319765.960244:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982897920:0@lo:38:kworker.0 00000100:00100000:3.0:1767319765.960437:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319765.960479:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898048 00000100:00100000:1.0:1767319765.960488:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898048 00000100:00080000:3.0:1767319765.960539:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319765.960610:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 814 00000100:00100000:1.0:1767319765.960617:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982898048:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319765.960643:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982898176:0@lo:400:kworker.0 00000100:00100000:1.0:1767319765.960676:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982898048:12345-0@lo:400:kworker.0 Request processed in 57us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319765.960684:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 814 00000100:00100000:3.0:1767319765.960749:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982898048:0@lo:400:kworker.0 00000100:00100000:0.0:1767319765.961006:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982897408:12345-0@lo:400:kworker.0 Request processed in 3487us (3777us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319765.961026:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1310 00010000:00080000:2.0:1767319765.961240:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0a80 x1853163982897408/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319781 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319765.961251:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982897408:0@lo:400:kworker.0 00010000:00010000:2.0:1767319766.149522:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319766.468714:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319766.468738:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982898432:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319766.468811:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319766.468898:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898432 00000100:00100000:0.0:1767319766.468906:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898432 00000100:00100000:0.0:1767319766.468959:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 886 00000100:00100000:0.0:1767319766.468969:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982898432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319766.468996:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319766.469038:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982898432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319766.469046:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 886 00000100:00100000:2.0:1767319766.469126:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982898432:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319766.469448:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982898560:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319766.469478:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319766.469601:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898560 00000100:00100000:2.0:1767319766.469605:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898560 00000100:00100000:2.0:1767319766.469638:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 887 00000100:00100000:2.0:1767319766.469645:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982898560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319766.469662:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319766.469703:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982898560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (224us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319766.469709:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 887 00000100:00100000:3.0:1767319766.469898:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982898560:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319766.925695:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319766.925802:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319766.925892:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643520 00000100:00100000:0.0:1767319766.925902:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643520 00000100:00100000:0.0:1767319766.925983:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643392 00000100:00100000:0.0:1767319766.925987:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643392 00000100:00100000:0.0:1767319766.926012:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1312 00000100:00100000:0.0:1767319766.926022:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915643520:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319766.926099:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915643520:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (422us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319766.926107:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1312 00000100:00080000:3.0:1767319766.926119:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319766.926129:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1730 00000100:00100000:0.0:1767319766.926137:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915643392:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319766.926158:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319766.926187:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915643392:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (384us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319766.926194:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1730 00010000:00010000:2.0:1767319767.173000:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319768.197605:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319769.220506:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319770.244462:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319771.076738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319771.076748:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319771.076803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319771.076807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319771.076810:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319771.076819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5014 00000100:00080000:3.0:1767319771.076823:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319771.076829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319771.076849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0700 x1853163982898816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319771.076884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319771.076887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319771.076889:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319771.076893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5014 00000100:00080000:3.0:1767319771.076896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319771.076899:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319771.076909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163982899072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319771.076918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319771.076921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319771.076923:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319771.076926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5014 00000100:00080000:3.0:1767319771.076928:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319771.076931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319771.076941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2d80 x1853163982899200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319771.076975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319771.076979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319771.076980:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319771.076985:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5014 00000100:00080000:3.0:1767319771.076987:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319771.076990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319771.077000:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c380 x1853163982899584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319771.077067:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982898816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319771.077147:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077181:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982899072:0@lo:38:kworker.0 00000100:00100000:2.0:1767319771.077201:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077217:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982899200:0@lo:38:kworker.0 00000100:00100000:2.0:1767319771.077237:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077252:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982899584:0@lo:38:kworker.0 00000100:00100000:2.0:1767319771.077301:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077329:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982898944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319771.077353:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077370:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319771.077375:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982899328:0@lo:400:kworker.0 00000100:00100000:2.0:1767319771.077398:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.077427:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898816 00000100:00100000:2.0:1767319771.077435:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898816 00000100:00100000:2.0:1767319771.077463:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899072 00000100:00100000:2.0:1767319771.077465:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899072 00000100:00100000:2.0:1767319771.077473:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899584 00000100:00100000:2.0:1767319771.077475:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899584 00000100:00100000:2.0:1767319771.077482:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898944 00000100:00100000:2.0:1767319771.077484:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898944 00000100:00100000:2.0:1767319771.077515:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1313 00000100:00100000:2.0:1767319771.077524:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982898944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319771.077569:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982898944:12345-0@lo:400:kworker.0 Request processed in 47us (215us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319771.077576:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1313 00000100:00100000:2.0:1767319771.077590:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1731 00000100:00100000:2.0:1767319771.077594:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982898816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319771.077624:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319771.077653:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982898816:12345-0@lo:38:kworker.0 Request processed in 57us (507us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319771.077660:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1731 00000100:00100000:2.0:1767319771.077671:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732 00000100:00100000:2.0:1767319771.077675:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982899072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319771.077684:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319771.077704:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982899072:12345-0@lo:38:kworker.0 Request processed in 28us (502us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319771.077709:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732 00000100:00100000:2.0:1767319771.077719:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1733 00000100:00100000:2.0:1767319771.077722:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982899584:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319771.077730:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319771.077749:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982899584:12345-0@lo:38:kworker.0 Request processed in 25us (478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319771.077754:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1733 00000100:00100000:2.0:1767319771.077771:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899200 00000100:00100000:2.0:1767319771.077775:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899200 00000100:00100000:2.0:1767319771.077791:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 458 00000100:00100000:2.0:1767319771.077796:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982899200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319771.077804:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319771.077825:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982899200:12345-0@lo:38:kworker.0 Request processed in 28us (587us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319771.077830:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 458 00000100:00100000:2.0:1767319771.077849:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899328 00000100:00100000:2.0:1767319771.077852:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899328 00000100:00100000:2.0:1767319771.077876:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 815 00000100:00100000:2.0:1767319771.077882:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982899328:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319771.077909:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982899328:12345-0@lo:400:kworker.0 Request processed in 28us (511us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319771.077915:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 815 00000100:00100000:2.0:1767319771.077953:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982899328:0@lo:400:kworker.0 00010000:00080000:2.0:1767319771.077964:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2680 x1853163982898944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319787 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319771.077972:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982898944:0@lo:400:kworker.0 00000100:00080000:2.0:1767319771.077999:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319771.078003:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319771.078006:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982898816:0@lo:38:kworker.0 00000100:00080000:2.0:1767319771.078014:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319771.078017:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319771.078019:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982899072:0@lo:38:kworker.0 00000100:00080000:2.0:1767319771.078026:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319771.078028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319771.078031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982899200:0@lo:38:kworker.0 00000100:00080000:2.0:1767319771.078037:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319771.078039:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319771.078042:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982899584:0@lo:38:kworker.0 00000100:00080000:3.0:1767319771.078321:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319771.078326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319771.078470:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982899456:0@lo:400:kworker.0 00000100:00100000:1.0:1767319771.078508:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319771.078539:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319771.078545:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982898688:0@lo:400:kworker.0 00000100:00100000:1.0:1767319771.078565:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319771.078596:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982898688 00000100:00100000:1.0:1767319771.078612:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982898688 00000100:00100000:1.0:1767319771.078650:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1314 00000100:00100000:1.0:1767319771.078657:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982898688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319771.078691:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982898688:12345-0@lo:400:kworker.0 Request processed in 35us (125us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319771.078697:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1314 00000100:00100000:1.0:1767319771.078720:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899456 00000100:00100000:1.0:1767319771.078723:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899456 00000100:00100000:1.0:1767319771.078745:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 816 00000100:00100000:1.0:1767319771.078751:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982899456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319771.078778:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982899456:12345-0@lo:400:kworker.0 Request processed in 27us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319771.078784:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 816 00010000:00080000:1.0:1767319771.078812:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1500 x1853163982898688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319787 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319771.078823:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982898688:0@lo:400:kworker.0 00000100:00100000:1.0:1767319771.078832:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982899456:0@lo:400:kworker.0 00000100:00080000:3.0:1767319771.078899:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319771.078913:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319771.269218:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319771.589170:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319771.589196:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982899712:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319771.589261:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319771.589546:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899712 00000100:00100000:2.0:1767319771.589555:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899712 00000100:00100000:2.0:1767319771.589674:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 888 00000100:00100000:2.0:1767319771.589684:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982899712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319771.589714:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319771.589780:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982899712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (518us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319771.589787:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 888 00000100:00100000:1.0:1767319771.589860:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982899712:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319771.590262:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982899840:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319771.590308:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319771.590767:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899840 00000100:00100000:3.0:1767319771.590774:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899840 00000100:00100000:2.0:1767319771.590823:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 889 00000100:00100000:2.0:1767319771.590831:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982899840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319771.590851:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319771.590901:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982899840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (591us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319771.590907:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 889 00000100:00100000:1.0:1767319771.590979:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982899840:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319772.043484:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319772.043539:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319772.044528:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643648 00000100:00100000:0.0:1767319772.044541:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643648 00000100:00100000:0.0:1767319772.044724:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643776 00000100:00100000:0.0:1767319772.044729:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643776 00000100:00100000:0.0:1767319772.044795:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1315 00000100:00100000:0.0:1767319772.044805:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915643776:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319772.044906:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915643776:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (1366us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319772.044914:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1315 00000100:00100000:0.0:1767319772.044944:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1734 00000100:00100000:0.0:1767319772.044948:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915643648:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319772.044970:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319772.044996:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915643648:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (1524us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319772.045003:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1734 00000100:00080000:3.0:1767319772.045315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319772.292984:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319773.316640:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319774.340470:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319775.364972:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319776.196557:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319776.196568:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319776.196661:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982899968:0@lo:400:kworker.0 00000100:00100000:2.0:1767319776.196734:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319776.196815:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319776.196819:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319776.196823:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319776.196829:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5019 00000100:00080000:3.0:1767319776.196832:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319776.196839:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319776.196862:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398fb80 x1853163982900096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319776.196895:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319776.196898:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319776.196901:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319776.196905:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5019 00000100:00080000:3.0:1767319776.196907:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319776.196911:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319776.196923:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982900352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319776.196933:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319776.196936:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319776.196938:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319776.196949:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982900224:0@lo:400:kworker.0 00000100:00080000:3.0:1767319776.196951:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5019 00000100:00080000:3.0:1767319776.196955:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319776.196961:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319776.196976:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d880 x1853163982900480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319776.196990:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319776.197016:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319776.197022:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982900608:0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197048:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319776.197051:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982899968 00000100:00100000:0.0:1767319776.197059:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982899968 00000100:00100000:1.0:1767319776.197078:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900224 00000100:00100000:1.0:1767319776.197084:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900224 00000100:00100000:0.0:1767319776.197117:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1316 00000100:00100000:1.0:1767319776.197125:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767319776.197137:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982900224:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319776.197139:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982899968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197196:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982900224:12345-0@lo:400:kworker.0 Request processed in 61us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319776.197203:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767319776.197226:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982900736:0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197247:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319776.197300:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900608 00000100:00100000:1.0:1767319776.197304:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900608 00000100:00100000:1.0:1767319776.197328:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900736 00000100:00100000:1.0:1767319776.197331:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900736 00000100:00100000:1.0:1767319776.197349:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 817 00000100:00100000:1.0:1767319776.197354:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982900608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197382:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982900608:12345-0@lo:400:kworker.0 Request processed in 28us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319776.197387:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 817 00000100:00100000:1.0:1767319776.197397:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 818 00000100:00100000:1.0:1767319776.197401:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982900736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197426:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982900736:12345-0@lo:400:kworker.0 Request processed in 24us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319776.197431:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 818 00000100:00100000:1.0:1767319776.197472:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982900736:0@lo:400:kworker.0 00000100:00100000:1.0:1767319776.197521:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982900608:0@lo:400:kworker.0 00010000:00080000:1.0:1767319776.197533:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f100 x1853163982900224/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319792 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319776.197541:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982900224:0@lo:400:kworker.0 00000100:00080000:1.0:1767319776.197571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319776.197575:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319776.197577:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319776.197587:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982900096:0@lo:38:kworker.0 00000100:00100000:1.0:1767319776.197611:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319776.197627:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982900352:0@lo:38:kworker.0 00000100:00100000:1.0:1767319776.197646:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319776.197662:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982900480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319776.197680:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319776.197713:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900480 00000100:00100000:1.0:1767319776.197717:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900480 00000100:00100000:1.0:1767319776.197736:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 459 00000100:00100000:1.0:1767319776.197740:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982900480:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319776.197767:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319776.197800:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982900480:12345-0@lo:38:kworker.0 Request processed in 59us (118us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319776.197807:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 459 00000100:00080000:1.0:1767319776.197849:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319776.197854:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319776.197857:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982900480:0@lo:38:kworker.0 00000100:00100000:1.0:1767319776.197874:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900096 00000100:00100000:1.0:1767319776.197877:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900096 00000100:00100000:1.0:1767319776.197886:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900352 00000100:00100000:1.0:1767319776.197888:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900352 00000100:00100000:1.0:1767319776.197894:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1735 00000100:00100000:1.0:1767319776.197899:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982900096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319776.197908:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319776.197936:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982900096:12345-0@lo:38:kworker.0 Request processed in 36us (324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319776.197942:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1735 00000100:00100000:1.0:1767319776.197953:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1736 00000100:00100000:1.0:1767319776.197957:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982900352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319776.197966:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319776.197988:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982900352:12345-0@lo:38:kworker.0 Request processed in 30us (341us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319776.197994:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1736 00000100:00080000:1.0:1767319776.198015:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319776.198019:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319776.198022:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982900096:0@lo:38:kworker.0 00000100:00080000:1.0:1767319776.198029:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319776.198032:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319776.198034:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982900352:0@lo:38:kworker.0 00000100:00080000:3.0:1767319776.201966:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319776.201978:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319776.201982:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319776.201990:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5019 00000100:00080000:3.0:1767319776.201994:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319776.202001:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319776.202050:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163982900864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319776.202726:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982900864:0@lo:38:kworker.0 00000100:00100000:1.0:1767319776.203018:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319776.203169:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900864 00000100:00100000:3.0:1767319776.203178:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900864 00000100:00100000:3.0:1767319776.203259:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1737 00000100:00100000:3.0:1767319776.203268:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982900864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319776.203313:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319776.203359:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982900864:12345-0@lo:38:kworker.0 Request processed in 90us (340us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319776.203366:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1737 00000100:00080000:1.0:1767319776.203458:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319776.203464:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319776.203467:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982900864:0@lo:38:kworker.0 00000100:00100000:0.0:1767319776.204071:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982899968:12345-0@lo:400:kworker.0 Request processed in 6943us (7335us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319776.204082:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1316 00010000:00080000:2.0:1767319776.206931:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f480 x1853163982899968/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319792 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319776.206951:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982899968:0@lo:400:kworker.0 00010000:00010000:2.0:1767319776.388543:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319776.708714:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319776.708746:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982900992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319776.708850:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319776.709110:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982900992 00000100:00100000:3.0:1767319776.709120:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982900992 00000100:00100000:3.0:1767319776.709235:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 890 00000100:00100000:3.0:1767319776.709247:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982900992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319776.709334:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319776.709397:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982900992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 152us (546us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319776.709405:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 890 00000100:00100000:1.0:1767319776.709439:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319776.709451:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982901120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319776.709480:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319776.712353:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982900992:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319776.714702:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901120 00000100:00100000:0.0:1767319776.714717:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901120 00000100:00100000:0.0:1767319776.715514:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 891 00000100:00100000:0.0:1767319776.715527:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982901120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319776.715565:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319776.715665:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982901120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 141us (6183us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319776.715674:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 891 00000100:00100000:1.0:1767319776.719380:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982901120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319777.164207:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319777.164379:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319777.164779:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644032 00000100:00100000:0.0:1767319777.164791:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644032 00000100:00100000:0.0:1767319777.164874:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915643904 00000100:00100000:0.0:1767319777.164878:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915643904 00000100:00100000:0.0:1767319777.164907:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1318 00000100:00100000:0.0:1767319777.164928:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915644032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319777.165010:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915644032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (815us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319777.165019:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1318 00000100:00100000:0.0:1767319777.165040:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1738 00000100:00100000:0.0:1767319777.165045:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915643904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319777.165066:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319777.165093:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915643904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (714us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319777.165099:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1738 00000100:00080000:1.0:1767319777.165461:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319777.413085:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319778.436612:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319779.460481:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319780.484464:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319781.316469:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319781.316481:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319781.316548:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319781.316552:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1767319781.316555:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319781.316566:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319781.316575:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5024 00000100:00080000:3.0:1767319781.316580:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767319781.316583:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982901248:0@lo:400:kworker.0 00000100:00080000:3.0:1767319781.316588:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319781.316628:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163982901376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319781.316664:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319781.316687:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319781.316692:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319781.316695:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319781.316700:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5024 00000100:00080000:3.0:1767319781.316702:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319781.316706:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319781.316719:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398fb80 x1853163982901632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319781.316728:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319781.316732:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319781.316734:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319781.316738:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5024 00000100:00080000:3.0:1767319781.316741:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319781.316745:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319781.316746:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901248 00000100:00100000:2.0:1767319781.316755:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901248 00000100:00100000:3.0:1767319781.316757:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d880 x1853163982901760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319781.316787:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319781.316792:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319781.316793:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319781.316811:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5024 00000100:00080000:3.0:1767319781.316814:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319781.316818:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319781.316831:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1319 00000100:00100000:3.0:1767319781.316833:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c700 x1853163982902144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319781.316845:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982901248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319781.316861:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982901888:0@lo:400:kworker.0 00000100:00100000:3.0:1767319781.316905:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319781.316909:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982901248:12345-0@lo:400:kworker.0 Request processed in 66us (243us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319781.316919:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1319 00000100:00100000:3.0:1767319781.316932:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319781.316958:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982901504:0@lo:400:kworker.0 00000100:00100000:2.0:1767319781.316965:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982902016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319781.316987:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319781.317001:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767319781.317040:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f480 x1853163982901248/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319797 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319781.317053:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982901248:0@lo:400:kworker.0 00000100:00100000:3.0:1767319781.317450:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901504 00000100:00100000:3.0:1767319781.317457:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901504 00000100:00100000:3.0:1767319781.317497:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1320 00000100:00100000:3.0:1767319781.317505:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982901504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319781.317547:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982901504:12345-0@lo:400:kworker.0 Request processed in 42us (560us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319781.317554:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1320 00010000:00080000:3.0:1767319781.317584:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398ea00 x1853163982901504/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319797 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319781.317595:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982901504:0@lo:400:kworker.0 00000100:00100000:1.0:1767319781.317620:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901888 00000100:00100000:1.0:1767319781.317626:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901888 00000100:00100000:1.0:1767319781.317670:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902016 00000100:00100000:1.0:1767319781.317673:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902016 00000100:00100000:1.0:1767319781.317693:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 819 00000100:00100000:1.0:1767319781.317701:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982901888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319781.317739:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982901888:12345-0@lo:400:kworker.0 Request processed in 39us (834us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319781.317745:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 819 00000100:00100000:1.0:1767319781.317770:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 820 00000100:00100000:1.0:1767319781.317776:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982902016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319781.317819:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982902016:12345-0@lo:400:kworker.0 Request processed in 43us (819us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319781.317825:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 820 00000100:00080000:1.0:1767319781.317854:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319781.317857:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319781.317860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319781.317881:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982902016:0@lo:400:kworker.0 00000100:00100000:0.0:1767319781.318182:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982901376:0@lo:38:kworker.0 00000100:00100000:3.0:1767319781.318203:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982901888:0@lo:400:kworker.0 00000100:00100000:0.0:1767319781.318221:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319781.318257:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982901632:0@lo:38:kworker.0 00000100:00100000:0.0:1767319781.318313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319781.318332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982901760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319781.318354:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319781.318378:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982902144:0@lo:38:kworker.0 00000100:00100000:0.0:1767319781.318399:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319781.318437:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901760 00000100:00100000:0.0:1767319781.318441:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901760 00000100:00100000:0.0:1767319781.318466:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 460 00000100:00100000:0.0:1767319781.318471:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982901760:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319781.318489:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319781.318524:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982901760:12345-0@lo:38:kworker.0 Request processed in 51us (168us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319781.318529:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 460 00000100:00080000:0.0:1767319781.318557:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319781.318562:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319781.318579:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982901760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319781.318602:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901376 00000100:00100000:0.0:1767319781.318605:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901376 00000100:00100000:0.0:1767319781.318620:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982901632 00000100:00100000:0.0:1767319781.318621:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982901632 00000100:00100000:0.0:1767319781.318627:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902144 00000100:00100000:0.0:1767319781.318630:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902144 00000100:00100000:0.0:1767319781.318638:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1739 00000100:00100000:0.0:1767319781.318653:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982901376:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319781.318662:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319781.318692:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982901376:12345-0@lo:38:kworker.0 Request processed in 38us (472us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319781.318698:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1739 00000100:00100000:0.0:1767319781.318709:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740 00000100:00100000:0.0:1767319781.318714:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982901632:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319781.318722:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319781.318745:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982901632:12345-0@lo:38:kworker.0 Request processed in 31us (432us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319781.318751:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740 00000100:00100000:0.0:1767319781.318774:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1741 00000100:00100000:0.0:1767319781.318778:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982902144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319781.319021:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319781.319048:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982902144:12345-0@lo:38:kworker.0 Request processed in 268us (648us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319781.319054:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1741 00000100:00080000:0.0:1767319781.319078:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319781.319082:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319781.319086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982901376:0@lo:38:kworker.0 00000100:00080000:0.0:1767319781.319094:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319781.319097:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319781.319100:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982901632:0@lo:38:kworker.0 00000100:00080000:0.0:1767319781.319107:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319781.319110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319781.319112:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982902144:0@lo:38:kworker.0 00010000:00010000:2.0:1767319781.509733:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319781.828908:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982902272:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319781.828999:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319781.829084:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902272 00000100:00100000:1.0:1767319781.829091:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902272 00000100:00100000:1.0:1767319781.829140:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 892 00000100:00100000:1.0:1767319781.829150:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982902272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319781.829180:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319781.829225:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982902272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (225us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319781.829232:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 892 00000100:00100000:1.0:1767319781.829298:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982902272:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319781.831227:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319781.831243:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982902400:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319781.831333:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319781.831581:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902400 00000100:00100000:0.0:1767319781.831590:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902400 00000100:00100000:0.0:1767319781.831651:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 893 00000100:00100000:0.0:1767319781.831661:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982902400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319781.831694:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319781.831749:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982902400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (415us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319781.831768:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 893 00000100:00100000:0.0:1767319781.832167:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982902400:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319782.285229:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319782.285382:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644160 00000100:00100000:3.0:1767319782.285402:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644160 00000100:00100000:3.0:1767319782.285451:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1742 00000100:00100000:3.0:1767319782.285469:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319782.285495:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319782.285568:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 107us (349us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319782.285577:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1742 00000100:00100000:0.0:1767319782.289935:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319782.290148:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644288 00000100:00100000:3.0:1767319782.290161:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644288 00000100:00100000:3.0:1767319782.290261:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1321 00000100:00080000:1.0:1767319782.290366:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319782.290449:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915644288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319782.290717:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915644288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 448us (797us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319782.290724:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1321 00010000:00010000:2.0:1767319782.533749:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319783.558178:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319784.580961:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319785.605641:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319786.437661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319786.437675:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319786.437749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319786.437754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319786.437759:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319786.437770:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5029 00000100:00080000:3.0:1767319786.437776:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319786.437783:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319786.437809:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982902656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319786.437869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319786.437872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319786.437874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319786.437878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5029 00000100:00080000:3.0:1767319786.437881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319786.437885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319786.437898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398fb80 x1853163982902912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319786.437906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319786.437910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319786.437912:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319786.437916:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5029 00000100:00080000:3.0:1767319786.437918:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319786.437922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319786.437934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163982903040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319786.437955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319786.437959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319786.437961:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319786.437964:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5029 00000100:00080000:3.0:1767319786.437966:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319786.437969:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319786.437985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398f480 x1853163982903424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319786.438023:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982902784:0@lo:400:kworker.0 00000100:00100000:3.0:1767319786.438097:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319786.438164:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319786.438170:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982903168:0@lo:400:kworker.0 00000100:00100000:3.0:1767319786.438197:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319786.438231:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902784 00000100:00100000:3.0:1767319786.438240:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902784 00000100:00100000:3.0:1767319786.438327:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1322 00000100:00100000:3.0:1767319786.438338:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982902784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319786.438390:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982902784:12345-0@lo:400:kworker.0 Request processed in 54us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319786.438400:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1322 00010000:00080000:3.0:1767319786.438440:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398c700 x1853163982902784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319802 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319786.438453:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982902784:0@lo:400:kworker.0 00000100:00100000:2.0:1767319786.438534:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982903296:0@lo:400:kworker.0 00000100:00100000:2.0:1767319786.438557:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319786.438578:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319786.438581:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982902528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319786.438594:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319786.438615:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902528 00000100:00100000:2.0:1767319786.438617:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902528 00000100:00100000:2.0:1767319786.438639:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1323 00000100:00100000:2.0:1767319786.438646:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982902528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319786.438679:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982902528:12345-0@lo:400:kworker.0 Request processed in 34us (83us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319786.438685:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1323 00010000:00080000:2.0:1767319786.438708:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f100 x1853163982902528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319802 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319786.438715:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982902528:0@lo:400:kworker.0 00000100:00080000:1.0:1767319786.438866:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319786.438881:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982902656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319786.438908:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319786.438944:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982902912:0@lo:38:kworker.0 00000100:00100000:1.0:1767319786.438964:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319786.438981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982903040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319786.439005:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319786.439028:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982903424:0@lo:38:kworker.0 00000100:00100000:1.0:1767319786.439049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319786.439073:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902656 00000100:00100000:1.0:1767319786.439077:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902656 00000100:00100000:1.0:1767319786.439097:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982902912 00000100:00100000:1.0:1767319786.439099:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982902912 00000100:00100000:1.0:1767319786.439110:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903424 00000100:00100000:1.0:1767319786.439112:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903424 00000100:00100000:1.0:1767319786.439121:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1743 00000100:00100000:1.0:1767319786.439125:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982902656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319786.439141:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319786.439172:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982902656:12345-0@lo:38:kworker.0 Request processed in 46us (263us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319786.439178:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1743 00000100:00100000:1.0:1767319786.439191:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744 00000100:00100000:1.0:1767319786.439194:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982902912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319786.439203:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319786.439226:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982902912:12345-0@lo:38:kworker.0 Request processed in 30us (261us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319786.439231:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744 00000100:00100000:1.0:1767319786.439242:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1745 00000100:00100000:1.0:1767319786.439246:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982903424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319786.439254:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319786.439306:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982903424:12345-0@lo:38:kworker.0 Request processed in 58us (256us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319786.439311:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1745 00000100:00100000:1.0:1767319786.439342:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903168 00000100:00100000:1.0:1767319786.439346:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903168 00000100:00100000:1.0:1767319786.439374:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903296 00000100:00100000:1.0:1767319786.439377:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903296 00000100:00100000:1.0:1767319786.439393:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 821 00000100:00100000:1.0:1767319786.439399:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982903168:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319786.439431:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982903168:12345-0@lo:400:kworker.0 Request processed in 33us (1233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319786.439437:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 821 00000100:00100000:1.0:1767319786.439447:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 822 00000100:00100000:1.0:1767319786.439453:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982903296:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319786.439481:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982903296:12345-0@lo:400:kworker.0 Request processed in 29us (922us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319786.439487:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 822 00000100:00100000:2.0:1767319786.439621:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982903296:0@lo:400:kworker.0 00000100:00080000:2.0:1767319786.439670:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319786.439675:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319786.439679:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982902656:0@lo:38:kworker.0 00000100:00080000:2.0:1767319786.439689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319786.439693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319786.439696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982902912:0@lo:38:kworker.0 00000100:00080000:2.0:1767319786.439704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319786.439707:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319786.439709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982903424:0@lo:38:kworker.0 00000100:00080000:2.0:1767319786.439747:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319786.439751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319786.440706:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982903168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319786.441751:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903040 00000100:00100000:0.0:1767319786.441766:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903040 00000100:00100000:0.0:1767319786.441821:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 461 00000100:00100000:0.0:1767319786.441832:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982903040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319786.441858:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319786.441956:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982903040:12345-0@lo:38:kworker.0 Request processed in 124us (2949us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319786.441968:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 461 00000100:00080000:2.0:1767319786.442398:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319786.442405:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319786.442409:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982903040:0@lo:38:kworker.0 00010000:00010000:1.0:1767319786.628596:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319786.950448:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319786.950483:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319786.950483:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982903552:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319786.950498:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982903680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319786.950577:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319786.950621:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319786.950654:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903680 00000100:00100000:0.0:1767319786.950663:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903680 00000100:00100000:0.0:1767319786.950710:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903552 00000100:00100000:0.0:1767319786.950713:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903552 00000100:00100000:0.0:1767319786.950729:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 894 00000100:00100000:0.0:1767319786.950738:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982903680:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319786.950757:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 895 00000100:00100000:2.0:1767319786.950769:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982903552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319786.950771:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:2.0:1767319786.950794:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319786.950822:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982903680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319786.950829:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 894 00000100:00100000:0.0:1767319786.950872:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982903680:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319786.951121:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982903552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 355us (499us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319786.951128:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 895 00000100:00100000:0.0:1767319786.956041:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982903552:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319787.406443:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319787.406939:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915644416 offset 0 length 520: 4 00000100:00100000:0.0:1767319787.406974:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319787.407125:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644672 00000100:00100000:2.0:1767319787.407138:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644672 00000100:00100000:2.0:1767319787.407226:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644544 00000100:00100000:2.0:1767319787.407230:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644544 00000100:00100000:2.0:1767319787.407250:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1324 00000100:00100000:2.0:1767319787.407262:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915644672:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767319787.407367:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319787.407420:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915644672:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 160us (991us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319787.407429:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1324 00000100:00100000:2.0:1767319787.407462:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1746 00000100:00100000:2.0:1767319787.407468:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644544:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319787.407490:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319787.407529:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644544:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (553us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319787.407540:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1746 00010000:00010000:2.0:1767319787.652756:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319788.676508:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319789.700537:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319790.534610:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319765/real 1767319765] req@ffff9346392c0380 x1853163982897280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319790 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319790.534749:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346392c0380 x1853163982897280/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319790 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319790.534777:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319790.534868:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319790.534872:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982897280:0@lo:250:kworker.0 00000100:00080000:3.0:1767319790.535322:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.535329:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319790.535333:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.535338:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5014 00000100:00080000:3.0:1767319790.535343:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.535352:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319790.535414:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1500 x1853163982903808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319790.535474:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982903808:0@lo:250:kworker.0 00000400:00000100:1.0:1767319790.535549:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982903808 offset 0 length 520: 4 00000100:00080000:3.0:1767319790.535778:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.535782:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.535785:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.535789:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5034 00000100:00080000:3.0:1767319790.535792:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.535797:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319790.535813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3100 x1853163982904064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319790.535867:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319790.535882:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982903936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319790.535938:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319790.535979:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982903936 00000100:00100000:2.0:1767319790.535987:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982903936 00000100:00100000:2.0:1767319790.536059:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1325 00000100:00100000:2.0:1767319790.536070:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982903936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319790.536126:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982903936:12345-0@lo:400:kworker.0 Request processed in 58us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319790.536133:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1325 00010000:00080000:2.0:1767319790.536176:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0700 x1853163982903936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319806 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319790.536188:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982903936:0@lo:400:kworker.0 00000100:00080000:2.0:1767319790.536222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319790.536632:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982904064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319790.536740:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319790.536896:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.536902:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.536905:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.536916:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5034 00000100:00080000:3.0:1767319790.536920:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.536924:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319790.536939:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1f80 x1853163982904320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319790.536950:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.536953:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.536955:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.536958:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5034 00000100:00080000:3.0:1767319790.536961:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.536964:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319790.536975:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3800 x1853163982904448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319790.537160:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982904192:0@lo:400:kworker.0 00000100:00100000:0.0:1767319790.537193:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319790.537211:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319790.537214:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319790.537217:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319790.537220:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5034 00000100:00080000:3.0:1767319790.537223:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319790.537228:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319790.537253:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0a80 x1853163982904832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319790.537298:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982904704:0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.537335:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319790.537380:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982904576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.537404:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319790.537459:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904704 00000100:00100000:3.0:1767319790.537465:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904704 00000100:00100000:3.0:1767319790.537535:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904576 00000100:00100000:3.0:1767319790.537539:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904576 00000100:00100000:3.0:1767319790.537565:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 823 00000100:00100000:3.0:1767319790.537573:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982904704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.537619:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982904704:12345-0@lo:400:kworker.0 Request processed in 46us (283us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319790.537625:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 823 00000100:00100000:3.0:1767319790.537655:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 824 00000100:00100000:3.0:1767319790.537662:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982904576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.537695:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982904576:12345-0@lo:400:kworker.0 Request processed in 34us (290us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319790.537700:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 824 00000100:00080000:3.0:1767319790.537715:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319790.537719:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319790.537741:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982904704:0@lo:400:kworker.0 00000100:00100000:3.0:1767319790.537771:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982904576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319790.537847:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982904320:0@lo:38:kworker.0 00000100:00100000:2.0:1767319790.538538:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904064 00000100:00100000:2.0:1767319790.538548:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904064 00000100:00100000:3.0:1767319790.538608:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319790.538790:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982904448:0@lo:38:kworker.0 00000100:00100000:3.0:1767319790.538838:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319790.538857:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982904832:0@lo:38:kworker.0 00000100:00100000:3.0:1767319790.538881:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319790.538977:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904192 00000100:00100000:1.0:1767319790.538984:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904192 00000100:00100000:1.0:1767319790.539027:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904320 00000100:00100000:1.0:1767319790.539042:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904320 00000100:00100000:1.0:1767319790.539055:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1326 00000100:00100000:1.0:1767319790.539062:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982904192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319790.539115:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982904192:12345-0@lo:400:kworker.0 Request processed in 54us (1921us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319790.539122:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1326 00000100:00100000:1.0:1767319790.539138:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1747 00000100:00100000:1.0:1767319790.539142:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982904064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319790.539163:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319790.539192:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982904064:12345-0@lo:38:kworker.0 Request processed in 49us (2451us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319790.539199:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1747 00000100:00100000:1.0:1767319790.539210:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1748 00000100:00100000:1.0:1767319790.539213:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982904320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319790.539222:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319790.539256:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982904320:12345-0@lo:38:kworker.0 Request processed in 40us (648us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319790.539262:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1748 00000100:00100000:1.0:1767319790.539309:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904448 00000100:00100000:1.0:1767319790.539313:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904448 00000100:00100000:1.0:1767319790.539329:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 462 00000100:00100000:1.0:1767319790.539333:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982904448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319790.539342:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319790.539367:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982904448:12345-0@lo:38:kworker.0 Request processed in 33us (528us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319790.539374:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 462 00000100:00080000:1.0:1767319790.539393:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319790.539430:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0380 x1853163982904192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319806 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319790.539442:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982904192:0@lo:400:kworker.0 00000100:00080000:3.0:1767319790.540016:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319790.540023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319790.540036:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982904064:0@lo:38:kworker.0 00000100:00080000:3.0:1767319790.540062:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319790.540065:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319790.540068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982904320:0@lo:38:kworker.0 00000100:00080000:3.0:1767319790.540076:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319790.540079:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319790.540081:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982904448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319790.540234:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904832 00000100:00100000:2.0:1767319790.540250:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904832 00000100:00100000:2.0:1767319790.540293:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1749 00000100:00100000:2.0:1767319790.540299:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982904832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319790.540312:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319790.543410:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982904832:12345-0@lo:38:kworker.0 Request processed in 3095us (4512us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319790.543428:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1749 00000100:00080000:3.0:1767319790.543568:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319790.543574:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319790.543579:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982904832:0@lo:38:kworker.0 00010000:00010000:2.0:1767319790.724760:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319791.748490:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319792.073400:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982904960:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319792.073493:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319792.073721:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982905088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319792.073774:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319792.073824:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982904960 00000100:00100000:2.0:1767319792.073832:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982904960 00000100:00100000:2.0:1767319792.073896:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905088 00000100:00100000:2.0:1767319792.073899:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905088 00000100:00100000:2.0:1767319792.073931:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896 00000100:00100000:2.0:1767319792.073940:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346325d3850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982904960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319792.073989:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319792.074013:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 897 00000100:00100000:3.0:1767319792.074024:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982905088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319792.074051:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319792.074088:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982905088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (313us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319792.074096:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 897 00000100:00080000:0.0:1767319792.074106:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319792.074111:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319792.074138:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982905088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319792.080235:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346325d3850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982904960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 6273us (6720us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319792.080254:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896 00000100:00100000:1.0:1767319792.083751:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982904960:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319792.527479:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319792.527573:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319792.536738:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644928 00000100:00100000:2.0:1767319792.536754:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644928 00000100:00080000:1.0:1767319792.536881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319792.536909:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915644800 00000100:00100000:2.0:1767319792.536913:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915644800 00000100:00100000:3.0:1767319792.536966:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1327 00000100:00100000:3.0:1767319792.536981:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915644928:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319792.537077:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915644928:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (9659us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319792.537085:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1327 00000100:00100000:3.0:1767319792.537107:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1750 00000100:00100000:3.0:1767319792.537112:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644800:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319792.537135:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319792.537162:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915644800:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (9590us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319792.537168:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1750 00010000:00010000:1.0:1767319792.774254:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319793.796724:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319794.821427:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319795.652532:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.652546:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319795.652649:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.652664:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.652670:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.652680:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5039 00000100:00100000:0.0:1767319795.652682:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982905216:0@lo:400:kworker.0 00000100:00080000:2.0:1767319795.652686:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.652706:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319795.652735:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163982905344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319795.652771:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319795.652797:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.652802:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.652804:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.652808:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.652811:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.652816:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319795.652826:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905216 00000100:00100000:2.0:1767319795.652829:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7b80 x1853163982905600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319795.652835:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905216 00000100:00080000:2.0:1767319795.652843:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.652847:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.652849:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.652853:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5039 00000100:00080000:2.0:1767319795.652855:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.652859:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319795.652871:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5f80 x1853163982905728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319795.652904:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319795.652907:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319795.652910:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319795.652916:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5039 00000100:00100000:0.0:1767319795.652917:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1328 00000100:00080000:2.0:1767319795.652921:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319795.652926:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319795.652931:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982905216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.652956:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5180 x1853163982906112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319795.652980:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982905216:12345-0@lo:400:kworker.0 Request processed in 52us (208us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.652987:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982905472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319795.652990:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1328 00010000:00080000:0.0:1767319795.653029:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5500 x1853163982905216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319795.653043:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319795.653045:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982905216:0@lo:400:kworker.0 00000100:00100000:0.0:1767319795.653067:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319795.653075:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319795.653077:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982905984:0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.653086:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982905856:0@lo:400:kworker.0 00000100:00100000:0.0:1767319795.653109:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319795.653127:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905472 00000100:00100000:2.0:1767319795.653132:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905472 00000100:00100000:2.0:1767319795.653198:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1329 00000100:00100000:2.0:1767319795.653208:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982905472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319795.653297:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982905344:0@lo:38:kworker.0 00000100:00080000:3.0:1767319795.653418:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319795.653459:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905984 00000100:00100000:3.0:1767319795.653463:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905984 00000100:00100000:3.0:1767319795.653497:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 825 00000100:00100000:3.0:1767319795.653504:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982905984:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.662822:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982905472:12345-0@lo:400:kworker.0 Request processed in 9605us (9767us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.662842:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1329 00000100:00080000:2.0:1767319795.662929:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319795.662971:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767319795.663044:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee4380 x1853163982905472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319795.663060:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982905472:0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.663101:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905856 00000100:00100000:2.0:1767319795.663110:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905856 00000100:00100000:2.0:1767319795.665928:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 826 00000100:00100000:2.0:1767319795.665944:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982905856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319795.666050:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982905856:12345-0@lo:400:kworker.0 Request processed in 111us (3079us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319795.666057:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 826 00000100:00080000:2.0:1767319795.666086:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319795.666123:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982905856:0@lo:400:kworker.0 00000100:00100000:1.0:1767319795.673751:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319795.673814:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982905600:0@lo:38:kworker.0 00000100:00100000:1.0:1767319795.673852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319795.673899:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905344 00000100:00100000:1.0:1767319795.673912:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905344 00000100:00100000:1.0:1767319795.673960:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905600 00000100:00100000:1.0:1767319795.673963:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905600 00000100:00100000:1.0:1767319795.673983:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1751 00000100:00100000:1.0:1767319795.673990:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982905344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319795.674029:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319795.674074:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982905344:12345-0@lo:38:kworker.0 Request processed in 82us (334us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319795.674081:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1751 00000100:00100000:1.0:1767319795.674099:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752 00000100:00100000:1.0:1767319795.674102:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982905600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319795.674111:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319795.674137:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982905600:12345-0@lo:38:kworker.0 Request processed in 33us (285us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319795.674154:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752 00000100:00080000:1.0:1767319795.674205:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319795.674211:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319795.674215:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982905600:0@lo:38:kworker.0 00000100:00100000:1.0:1767319795.674224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982905728:0@lo:38:kworker.0 00000100:00100000:1.0:1767319795.674253:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319795.674316:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982905728 00000100:00100000:1.0:1767319795.674320:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982905728 00000100:00100000:1.0:1767319795.674341:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 463 00000100:00100000:1.0:1767319795.674346:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982905728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319795.674356:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319795.674399:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982905728:12345-0@lo:38:kworker.0 Request processed in 51us (144us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319795.674405:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 463 00000100:00080000:1.0:1767319795.674434:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319795.674438:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319795.674441:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982905728:0@lo:38:kworker.0 00000100:00100000:1.0:1767319795.674447:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982906112:0@lo:38:kworker.0 00000100:00100000:1.0:1767319795.674470:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319795.674490:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906112 00000100:00100000:1.0:1767319795.674495:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906112 00000100:00100000:1.0:1767319795.674511:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1753 00000100:00100000:1.0:1767319795.674515:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982906112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319795.674524:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319795.674549:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982906112:12345-0@lo:38:kworker.0 Request processed in 33us (78us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319795.674554:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1753 00000100:00080000:1.0:1767319795.674576:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319795.674579:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319795.674582:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982906112:0@lo:38:kworker.0 00000100:00080000:1.0:1767319795.674641:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319795.674646:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319795.674649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982905344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319795.679551:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982905984:12345-0@lo:400:kworker.0 Request processed in 26032us (26426us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319795.679572:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 825 00000100:00100000:0.0:1767319795.680157:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982905984:0@lo:400:kworker.0 00010000:00010000:1.0:1767319795.844874:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319796.869027:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319797.188756:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982906240:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319797.188858:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319797.189092:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906240 00000100:00100000:1.0:1767319797.189103:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906240 00000100:00100000:1.0:1767319797.189355:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 898 00000100:00100000:1.0:1767319797.189366:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982906240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319797.189404:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319797.189483:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982906240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (624us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319797.189491:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 898 00000100:00080000:2.0:1767319797.189543:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319797.194749:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982906368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319797.194839:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319797.201036:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906368 00000100:00100000:0.0:1767319797.201055:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906368 00000100:00100000:0.0:1767319797.201137:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 899 00000100:00100000:0.0:1767319797.201149:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982906368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319797.201207:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319797.201296:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982906368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 150us (6457us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319797.201304:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899 00000100:00100000:0.0:1767319797.201391:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982906240:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319797.201977:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319797.202012:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982906368:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319797.645632:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319797.645690:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319797.645781:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645056 00000100:00100000:1.0:1767319797.645786:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645184 00000100:00100000:1.0:1767319797.645792:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645184 00000100:00100000:2.0:1767319797.645792:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645056 00000100:00100000:2.0:1767319797.645858:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1330 00000100:00100000:1.0:1767319797.645865:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1754 00000100:00100000:2.0:1767319797.645869:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915645184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319797.645870:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915645056:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319797.645908:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319797.646120:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915645184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 252us (428us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319797.646128:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1330 00000100:00100000:1.0:1767319797.646620:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915645056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 746us (999us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319797.646628:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1754 00000100:00080000:3.0:1767319797.646735:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319797.892630:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319798.917145:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319799.940575:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319800.772774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319800.772799:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319800.772863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319800.772867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319800.772871:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319800.772879:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5044 00000100:00080000:3.0:1767319800.772884:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319800.772891:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319800.772921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0000 x1853163982906624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319800.772971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319800.772975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319800.772976:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319800.772981:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5044 00000100:00080000:3.0:1767319800.772983:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319800.772997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319800.773010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0700 x1853163982906880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319800.773019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319800.773022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319800.773023:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319800.773036:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5044 00000100:00080000:3.0:1767319800.773039:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319800.773081:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319800.773104:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c000 x1853163982907008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319800.773112:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319800.773132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319800.773136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319800.773137:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319800.773141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5044 00000100:00080000:3.0:1767319800.773143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319800.773150:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982907264:0@lo:400:kworker.0 00000100:00080000:3.0:1767319800.773153:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319800.773163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163982907392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319800.773185:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982906752:0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.773233:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319800.773233:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319800.773303:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906752 00000100:00100000:1.0:1767319800.773312:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906752 00000100:00100000:1.0:1767319800.773404:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1331 00000100:00100000:1.0:1767319800.773413:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982906752:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319800.773529:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907264 00000100:00100000:2.0:1767319800.773543:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907264 00000100:00100000:2.0:1767319800.773579:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 827 00000100:00100000:2.0:1767319800.773586:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982907264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.780676:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982906496:0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.780754:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.782063:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982907136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.782129:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.782190:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907136 00000100:00100000:0.0:1767319800.782200:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907136 00000100:00100000:0.0:1767319800.782285:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 828 00000100:00100000:0.0:1767319800.782295:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982907136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.782357:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982907136:12345-0@lo:400:kworker.0 Request processed in 62us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319800.782366:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 828 00000100:00100000:0.0:1767319800.782407:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982907136:0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.783572:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982906624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319800.783686:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.783722:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982906880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319800.783744:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.783761:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982907008:0@lo:38:kworker.0 00000100:00100000:0.0:1767319800.783810:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.783843:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982907392:0@lo:38:kworker.0 00000100:00100000:0.0:1767319800.783884:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319800.783940:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907008 00000100:00100000:0.0:1767319800.783947:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907008 00000100:00100000:0.0:1767319800.783978:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 464 00000100:00100000:0.0:1767319800.783985:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982907008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319800.784005:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319800.784046:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982907008:12345-0@lo:38:kworker.0 Request processed in 60us (234us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319800.784064:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 464 00000100:00100000:0.0:1767319800.784084:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906496 00000100:00100000:0.0:1767319800.784088:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906496 00000100:00100000:0.0:1767319800.784113:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906624 00000100:00100000:0.0:1767319800.784116:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906624 00000100:00100000:0.0:1767319800.784125:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982906880 00000100:00100000:0.0:1767319800.784128:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982906880 00000100:00100000:0.0:1767319800.784134:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907392 00000100:00100000:0.0:1767319800.784136:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907392 00000100:00100000:0.0:1767319800.784145:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1332 00000100:00100000:0.0:1767319800.784151:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982906496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.784196:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982906496:12345-0@lo:400:kworker.0 Request processed in 45us (3442us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319800.784202:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1332 00000100:00100000:0.0:1767319800.784215:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1755 00000100:00100000:0.0:1767319800.784219:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982906624:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319800.784228:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319800.784252:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982906624:12345-0@lo:38:kworker.0 Request processed in 31us (566us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319800.784258:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1755 00000100:00100000:0.0:1767319800.784268:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756 00000100:00100000:0.0:1767319800.784291:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982906880:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319800.784299:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319800.784322:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982906880:12345-0@lo:38:kworker.0 Request processed in 31us (577us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319800.784327:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756 00000100:00100000:0.0:1767319800.784338:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1757 00000100:00100000:0.0:1767319800.784342:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982907392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319800.784349:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319800.784384:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982907392:12345-0@lo:38:kworker.0 Request processed in 41us (499us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319800.784390:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1757 00000100:00100000:1.0:1767319800.784960:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982906752:12345-0@lo:400:kworker.0 Request processed in 11546us (11723us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319800.784970:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1331 00000100:00080000:1.0:1767319800.784995:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319800.785000:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319800.785003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319800.785039:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319800.785044:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319800.785048:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982906624:0@lo:38:kworker.0 00000100:00080000:1.0:1767319800.785068:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319800.785072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319800.785075:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982906880:0@lo:38:kworker.0 00000100:00080000:1.0:1767319800.785083:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319800.785086:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319800.785088:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982907008:0@lo:38:kworker.0 00000100:00080000:1.0:1767319800.785096:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319800.785099:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319800.785102:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982907392:0@lo:38:kworker.0 00000100:00100000:2.0:1767319800.786420:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982907264:12345-0@lo:400:kworker.0 Request processed in 12826us (13178us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319800.786432:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 827 00010000:00080000:3.0:1767319800.786819:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0380 x1853163982906752/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319816 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319800.786834:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982906752:0@lo:400:kworker.0 00010000:00080000:0.0:1767319800.787136:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3100 x1853163982906496/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319816 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319800.787152:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982906496:0@lo:400:kworker.0 00000100:00100000:0.0:1767319800.787180:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982907264:0@lo:400:kworker.0 00010000:00010000:2.0:1767319800.964691:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319801.988695:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319802.308695:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982907520:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319802.308813:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319802.309087:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907520 00000100:00100000:2.0:1767319802.309095:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907520 00000100:00080000:1.0:1767319802.309129:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319802.309147:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900 00000100:00100000:2.0:1767319802.309155:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982907520:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319802.309200:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319802.309261:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982907520:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (447us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319802.309268:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900 00000100:00100000:3.0:1767319802.309288:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982907648:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319802.309316:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319802.309414:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982907520:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319802.310242:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907648 00000100:00100000:2.0:1767319802.310251:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907648 00000100:00100000:2.0:1767319802.310322:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 901 00000100:00100000:2.0:1767319802.310329:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982907648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319802.310349:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319802.310403:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982907648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (1086us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319802.310409:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 901 00000100:00080000:1.0:1767319802.310468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319802.310760:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982907648:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319802.765152:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319802.765240:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319802.765377:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645440 00000100:00100000:2.0:1767319802.765389:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645440 00000100:00100000:2.0:1767319802.765527:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645312 00000100:00100000:2.0:1767319802.765541:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645312 00000100:00100000:2.0:1767319802.765564:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1333 00000100:00100000:2.0:1767319802.765574:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915645440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319802.765650:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915645440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (510us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319802.765658:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1333 00000100:00100000:2.0:1767319802.765675:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1758 00000100:00100000:2.0:1767319802.765680:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915645312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319802.765703:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319802.765729:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915645312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (489us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319802.768621:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1758 00000100:00080000:1.0:1767319802.769502:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319803.012674:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319804.036617:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319805.060479:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319805.892646:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319805.892657:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319805.892725:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319805.892729:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319805.892733:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319805.892742:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5049 00000100:00080000:0.0:1767319805.892746:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319805.892765:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319805.892788:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f480 x1853163982907904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319805.892828:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319805.892832:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319805.892833:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319805.892837:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5049 00000100:00080000:0.0:1767319805.892839:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319805.892842:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319805.892853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ea00 x1853163982908160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319805.892861:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319805.892864:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319805.892865:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319805.892869:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5049 00000100:00080000:0.0:1767319805.892871:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319805.892874:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319805.892885:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163982908288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319805.892904:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319805.892907:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319805.892909:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319805.892912:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5049 00000100:00080000:0.0:1767319805.892914:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319805.892917:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319805.892926:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982908672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319805.892953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982908416:0@lo:400:kworker.0 00000100:00100000:1.0:1767319805.893071:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982907904:0@lo:38:kworker.0 00000100:00100000:0.0:1767319805.893088:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319805.893162:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908416 00000100:00100000:2.0:1767319805.893170:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908416 00000100:00100000:2.0:1767319805.893229:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 829 00000100:00100000:2.0:1767319805.893239:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982908416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319805.893310:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982908544:0@lo:400:kworker.0 00000100:00100000:0.0:1767319805.899541:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319805.899563:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982908032:0@lo:400:kworker.0 00000100:00100000:0.0:1767319805.899623:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.899681:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908032 00000100:00100000:0.0:1767319805.899689:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908032 00000100:00100000:0.0:1767319805.899741:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1334 00000100:00100000:0.0:1767319805.899752:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982908032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319805.899797:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.899804:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982908032:12345-0@lo:400:kworker.0 Request processed in 55us (181us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319805.899811:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1334 00000100:00100000:0.0:1767319805.899825:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907904 00000100:00100000:0.0:1767319805.899827:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907904 00000100:00100000:1.0:1767319805.899827:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982908160:0@lo:38:kworker.0 00000100:00100000:0.0:1767319805.899847:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1759 00000100:00100000:1.0:1767319805.899848:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.899851:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982907904:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319805.899863:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982908288:0@lo:38:kworker.0 00010000:00020000:0.0:1767319805.899870:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319805.899899:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982907904:12345-0@lo:38:kworker.0 Request processed in 47us (102us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319805.899901:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.899906:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1759 00000100:00100000:0.0:1767319805.899915:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908160 00000100:00100000:0.0:1767319805.899917:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908160 00000100:00100000:1.0:1767319805.899923:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982908672:0@lo:38:kworker.0 00000100:00100000:0.0:1767319805.899927:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1760 00000100:00100000:0.0:1767319805.899931:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982908160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319805.899940:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319805.899944:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.899979:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982908160:12345-0@lo:38:kworker.0 Request processed in 47us (130us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319805.899984:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1760 00000100:00080000:1.0:1767319805.899988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319805.899991:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908672 00000100:00080000:1.0:1767319805.899993:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319805.899994:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908672 00000100:00100000:1.0:1767319805.899996:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982907904:0@lo:38:kworker.0 00000100:00100000:0.0:1767319805.900003:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1761 00000100:00080000:1.0:1767319805.900005:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319805.900007:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982908672:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767319805.900008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319805.900011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982908160:0@lo:38:kworker.0 00010000:00020000:0.0:1767319805.900015:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767319805.900035:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319805.900036:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982908672:12345-0@lo:38:kworker.0 Request processed in 28us (91us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319805.900041:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1761 00000100:00080000:1.0:1767319805.900052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319805.900055:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982908672:0@lo:38:kworker.0 00000100:00100000:1.0:1767319805.900072:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908288 00010000:00080000:0.0:1767319805.900075:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d500 x1853163982908032/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319805.900075:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908288 00000100:00100000:0.0:1767319805.900087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982908032:0@lo:400:kworker.0 00000100:00100000:1.0:1767319805.900521:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 465 00000100:00100000:1.0:1767319805.900527:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982908288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319805.900538:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319805.900573:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982908288:12345-0@lo:38:kworker.0 Request processed in 44us (670us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319805.900578:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 465 00000100:00080000:1.0:1767319805.900610:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319805.900614:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319805.900617:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982908288:0@lo:38:kworker.0 00000100:00080000:1.0:1767319805.900656:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319805.900662:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319805.901172:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982907776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319805.901210:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319805.901225:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982907776 00000100:00100000:0.0:1767319805.901230:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982907776 00000100:00080000:1.0:1767319805.901267:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319805.901293:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1335 00000100:00100000:0.0:1767319805.901300:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982907776:12345-0@lo:400:kworker.0 00010000:00080000:1.0:1767319805.901473:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396dc00 x1853163982907776/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319821 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319805.901485:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982907776:0@lo:400:kworker.0 00000100:00100000:0.0:1767319805.902235:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982907776:12345-0@lo:400:kworker.0 Request processed in 52us (140us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319805.902245:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1335 00000100:00100000:2.0:1767319805.902509:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982908416:12345-0@lo:400:kworker.0 Request processed in 9269us (9418us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319805.902518:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 829 00000100:00100000:0.0:1767319805.907659:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982908416:0@lo:400:kworker.0 00000100:00100000:3.0:1767319805.907817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319805.907948:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908544 00000100:00100000:2.0:1767319805.907966:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908544 00000100:00100000:2.0:1767319805.908050:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 830 00000100:00100000:2.0:1767319805.908061:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982908544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319805.908131:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982908544:12345-0@lo:400:kworker.0 Request processed in 72us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319805.908139:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 830 00000100:00100000:3.0:1767319805.910116:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982908544:0@lo:400:kworker.0 00000100:00080000:1.0:1767319805.910333:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319806.084637:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319807.108509:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319807.428699:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319807.428725:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982908800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319807.428795:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319807.428975:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908800 00000100:00100000:1.0:1767319807.428983:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908800 00000100:00100000:1.0:1767319807.429060:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902 00000100:00100000:1.0:1767319807.429070:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982908800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319807.429098:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319807.429141:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982908800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319807.429148:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902 00000100:00100000:1.0:1767319807.429748:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982908928:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319807.429783:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319807.429899:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319807.429952:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982908800:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319807.430139:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982908928 00000100:00100000:2.0:1767319807.430145:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982908928 00000100:00100000:2.0:1767319807.430203:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 903 00000100:00100000:2.0:1767319807.430212:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982908928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319807.430234:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319807.430310:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982908928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (526us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319807.430316:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 903 00000100:00100000:2.0:1767319807.430350:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982908928:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319807.430999:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319807.887101:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319807.887330:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319807.892465:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645568 00000100:00100000:3.0:1767319807.892488:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645568 00000100:00100000:3.0:1767319807.892607:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645696 00000100:00100000:3.0:1767319807.892612:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645696 00000100:00080000:2.0:1767319807.892715:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319807.894151:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1336 00000100:00100000:1.0:1767319807.894173:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915645696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319807.895678:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915645696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1496us (8329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319807.895692:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1336 00000100:00100000:1.0:1767319807.895732:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1762 00000100:00100000:1.0:1767319807.895738:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915645568:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319807.895761:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319807.895815:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915645568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 75us (8728us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319807.895821:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1762 00010000:00010000:1.0:1767319808.132973:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319809.158408:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319810.180534:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319811.012421:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.012434:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319811.012504:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.012509:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.012513:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.012524:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.012530:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.012538:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319811.012564:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163982909184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.012616:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.012619:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.012621:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.012625:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.012627:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.012631:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319811.012642:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c380 x1853163982909440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.012651:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.012654:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.012656:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.012659:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.012661:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.012665:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319811.012682:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163982909568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319811.012702:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319811.012706:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319811.012708:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319811.012711:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5054 00000100:00080000:0.0:1767319811.012713:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319811.012716:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319811.012724:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163982909952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319811.012758:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982909312:0@lo:400:kworker.0 00000100:00100000:0.0:1767319811.012830:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319811.012899:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319811.012906:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982909696:0@lo:400:kworker.0 00000100:00100000:0.0:1767319811.012931:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319811.012964:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909696 00000100:00100000:0.0:1767319811.012971:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909696 00000100:00100000:0.0:1767319811.013023:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767319811.013032:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982909696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319811.013078:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982909696:12345-0@lo:400:kworker.0 Request processed in 48us (146us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319811.013089:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 831 00000100:00100000:0.0:1767319811.013127:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982909696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319811.013552:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909312 00000100:00100000:1.0:1767319811.013559:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909312 00000100:00100000:1.0:1767319811.013591:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1337 00000100:00100000:1.0:1767319811.013599:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982909312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319811.013667:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982909312:12345-0@lo:400:kworker.0 Request processed in 69us (838us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319811.013682:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1337 00000100:00100000:1.0:1767319811.013702:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982909824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319811.013730:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.013756:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319811.013761:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982909056:0@lo:400:kworker.0 00000100:00100000:1.0:1767319811.013825:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319811.013854:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909056 00000100:00100000:1.0:1767319811.013858:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909056 00000100:00100000:1.0:1767319811.013898:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1338 00000100:00100000:1.0:1767319811.013903:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982909056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319811.013939:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982909056:12345-0@lo:400:kworker.0 Request processed in 36us (114us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319811.013945:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1338 00010000:00080000:1.0:1767319811.013972:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d880 x1853163982909312/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319827 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319811.013982:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982909312:0@lo:400:kworker.0 00010000:00080000:1.0:1767319811.014002:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ca80 x1853163982909056/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319827 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319811.014009:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982909056:0@lo:400:kworker.0 00000100:00080000:2.0:1767319811.014095:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319811.014101:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319811.014114:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909824 00000100:00100000:2.0:1767319811.014119:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909824 00000100:00100000:2.0:1767319811.014157:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 832 00000100:00100000:2.0:1767319811.014164:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982909824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319811.014203:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982909824:12345-0@lo:400:kworker.0 Request processed in 40us (472us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319811.014209:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 832 00000100:00100000:2.0:1767319811.014233:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982909824:0@lo:400:kworker.0 00000100:00100000:3.0:1767319811.014445:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982909184:0@lo:38:kworker.0 00000100:00100000:3.0:1767319811.014481:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.014517:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982909440:0@lo:38:kworker.0 00000100:00100000:3.0:1767319811.014539:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.014563:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982909568:0@lo:38:kworker.0 00000100:00100000:3.0:1767319811.014586:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.014607:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982909952:0@lo:38:kworker.0 00000100:00100000:3.0:1767319811.014626:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319811.014651:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909568 00000100:00100000:3.0:1767319811.014655:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909568 00000100:00100000:3.0:1767319811.014678:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 466 00000100:00100000:3.0:1767319811.014683:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982909568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319811.014702:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319811.014734:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982909568:12345-0@lo:38:kworker.0 Request processed in 49us (147us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319811.014740:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 466 00000100:00080000:3.0:1767319811.014769:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319811.014774:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319811.014777:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982909568:0@lo:38:kworker.0 00000100:00100000:2.0:1767319811.014847:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909184 00000100:00100000:2.0:1767319811.014852:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909184 00000100:00100000:2.0:1767319811.014891:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909440 00000100:00100000:2.0:1767319811.014895:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909440 00000100:00100000:2.0:1767319811.014906:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982909952 00000100:00100000:2.0:1767319811.014908:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982909952 00000100:00100000:2.0:1767319811.014921:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1763 00000100:00100000:2.0:1767319811.014926:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319811.014936:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319811.014970:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909184:12345-0@lo:38:kworker.0 Request processed in 43us (488us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319811.014979:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1763 00000100:00100000:2.0:1767319811.014991:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764 00000100:00100000:2.0:1767319811.014995:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319811.015003:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319811.015024:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909440:12345-0@lo:38:kworker.0 Request processed in 28us (485us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319811.015030:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764 00000100:00100000:2.0:1767319811.015040:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1765 00000100:00100000:2.0:1767319811.015044:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319811.015052:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319811.015075:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982909952:12345-0@lo:38:kworker.0 Request processed in 29us (447us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319811.015081:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1765 00000100:00080000:2.0:1767319811.015103:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319811.015107:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319811.015110:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982909184:0@lo:38:kworker.0 00000100:00080000:2.0:1767319811.015118:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319811.015121:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319811.015124:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982909440:0@lo:38:kworker.0 00000100:00080000:2.0:1767319811.015132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319811.015135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319811.015138:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982909952:0@lo:38:kworker.0 00000100:00080000:1.0:1767319811.015307:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767319811.204594:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319811.984249:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319811.984300:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915645824 offset 0 length 520: 4 00000100:00100000:0.0:1767319811.984328:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319811.987789:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646080 00000100:00100000:1.0:1767319811.987806:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646080 00000100:00100000:1.0:1767319811.987914:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915645952 00000100:00100000:1.0:1767319811.987918:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915645952 00000100:00080000:2.0:1767319811.988200:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319811.988308:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1339 00000100:00100000:0.0:1767319811.988322:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915646080:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319811.988417:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915646080:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (4176us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319811.988425:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1339 00000100:00100000:0.0:1767319811.988448:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1766 00000100:00100000:0.0:1767319811.988453:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915645952:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319811.988471:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319811.988493:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915645952:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 39us (4164us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319811.988499:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1766 00010000:00010000:2.0:1767319812.231678:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319812.548663:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319812.548695:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982910080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319812.548782:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319812.548942:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910080 00000100:00100000:1.0:1767319812.548962:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910080 00000100:00100000:1.0:1767319812.549221:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 904 00000100:00100000:1.0:1767319812.549233:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982910080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319812.549310:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319812.549492:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982910080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 261us (709us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319812.549501:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 904 00000100:00100000:0.0:1767319812.549622:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982910080:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319812.552771:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982910208:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319812.552867:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319812.553294:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910208 00000100:00100000:2.0:1767319812.553303:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910208 00000100:00100000:2.0:1767319812.553485:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905 00000100:00100000:2.0:1767319812.553507:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982910208:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319812.553545:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319812.553607:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982910208:12345-0@lo:13:osp-pre-0-1.0 Request processed in 112us (739us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319812.553615:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905 00000100:00100000:1.0:1767319812.553698:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982910208:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319813.253526:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319814.276587:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319815.108448:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319790/real 1767319790] req@ffff93463a8e1500 x1853163982903808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319815.108494:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e1500 x1853163982903808/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319815.108517:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319815.108543:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319815.108547:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982903808:0@lo:250:kworker.0 00000100:00080000:0.0:1767319815.108689:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.108697:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319815.108701:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.108707:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5039 00000100:00080000:0.0:1767319815.108711:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.108718:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319815.108756:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c380 x1853163982910336/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319815.108846:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.108850:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.108852:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.108855:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.108858:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.108862:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319815.108883:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163982910592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319815.108913:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.108916:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.108918:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.108921:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.108924:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.108927:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319815.108940:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ca80 x1853163982910848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319815.108949:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.108952:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.108954:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.108958:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.108960:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.108964:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319815.108975:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d880 x1853163982910976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319815.108995:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319815.108999:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319815.109001:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319815.109004:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5059 00000100:00080000:0.0:1767319815.109006:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319815.109009:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319815.109018:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163982911360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319815.109056:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982911104:0@lo:400:kworker.0 00000100:00100000:0.0:1767319815.109128:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319815.109192:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319815.109198:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982910720:0@lo:400:kworker.0 00000100:00100000:0.0:1767319815.109224:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319815.109257:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911104 00000100:00100000:0.0:1767319815.109266:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911104 00000100:00100000:0.0:1767319815.109345:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 833 00000100:00100000:0.0:1767319815.109354:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604770450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982911104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319815.109407:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604770450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982911104:12345-0@lo:400:kworker.0 Request processed in 54us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319815.109415:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 833 00000100:00080000:0.0:1767319815.111112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319815.111166:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982911104:0@lo:400:kworker.0 00000100:00100000:3.0:1767319815.111397:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910720 00000100:00100000:3.0:1767319815.111405:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910720 00000100:00100000:3.0:1767319815.111476:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1340 00000100:00100000:3.0:1767319815.111485:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982910720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319815.111565:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982910720:12345-0@lo:400:kworker.0 Request processed in 80us (2338us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319815.111571:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1340 00000100:00100000:3.0:1767319815.111599:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982910336:0@lo:250:kworker.0 00000400:00000100:3.0:1767319815.111624:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982910336 offset 0 length 520: 4 00000100:00100000:3.0:1767319815.111648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982910592:0@lo:38:kworker.0 00000100:00100000:3.0:1767319815.111674:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319815.111694:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982910848:0@lo:38:kworker.0 00000100:00100000:3.0:1767319815.111717:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319815.111736:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982910976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319815.111758:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319815.111773:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982911360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319815.111804:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319815.111837:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910592 00000100:00100000:3.0:1767319815.111842:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910592 00000100:00100000:3.0:1767319815.111859:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910848 00000100:00100000:3.0:1767319815.111861:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910848 00000100:00100000:3.0:1767319815.111868:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911360 00000100:00100000:3.0:1767319815.111870:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911360 00000100:00100000:3.0:1767319815.111878:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1767 00000100:00100000:3.0:1767319815.111883:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982910592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319815.111902:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319815.111934:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982910592:12345-0@lo:38:kworker.0 Request processed in 49us (260us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319815.111941:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1767 00000100:00100000:3.0:1767319815.111952:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768 00000100:00100000:3.0:1767319815.111955:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982910848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319815.111964:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319815.111989:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982910848:12345-0@lo:38:kworker.0 Request processed in 31us (271us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319815.111995:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768 00000100:00100000:3.0:1767319815.112006:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1769 00000100:00100000:3.0:1767319815.112009:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982911360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319815.112016:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319815.112052:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982911360:12345-0@lo:38:kworker.0 Request processed in 41us (247us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319815.112058:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1769 00000100:00100000:3.0:1767319815.112077:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910976 00000100:00100000:3.0:1767319815.112080:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910976 00000100:00100000:3.0:1767319815.112100:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 467 00000100:00100000:3.0:1767319815.112105:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982910976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319815.112113:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319815.112137:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982910976:12345-0@lo:38:kworker.0 Request processed in 32us (379us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319815.112142:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 467 00010000:00080000:3.0:1767319815.112183:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ed80 x1853163982910720/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319815.112195:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982910720:0@lo:400:kworker.0 00000100:00080000:3.0:1767319815.112219:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319815.112224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319815.112227:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982910592:0@lo:38:kworker.0 00000100:00080000:3.0:1767319815.112235:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319815.112239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319815.112241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982910848:0@lo:38:kworker.0 00000100:00080000:3.0:1767319815.112249:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319815.112252:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319815.112255:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982910976:0@lo:38:kworker.0 00000100:00080000:3.0:1767319815.112262:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319815.112266:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319815.112301:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982911360:0@lo:38:kworker.0 00000100:00100000:2.0:1767319815.112489:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982911232:0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.112526:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.112562:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319815.112570:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982910464:0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.112592:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319815.112616:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982910464 00000100:00100000:2.0:1767319815.112620:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982910464 00000100:00100000:2.0:1767319815.112651:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1341 00000100:00100000:2.0:1767319815.112657:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982910464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319815.112696:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982910464:12345-0@lo:400:kworker.0 Request processed in 38us (102us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319815.112702:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1341 00010000:00080000:2.0:1767319815.112730:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396fb80 x1853163982910464/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319815.112740:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982910464:0@lo:400:kworker.0 00000100:00100000:3.0:1767319815.113006:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911232 00000100:00100000:3.0:1767319815.113011:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911232 00000100:00100000:3.0:1767319815.113059:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 834 00000100:00100000:3.0:1767319815.113066:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982911232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319815.113116:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982911232:12345-0@lo:400:kworker.0 Request processed in 50us (589us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319815.113122:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 834 00000100:00100000:2.0:1767319815.113246:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982911232:0@lo:400:kworker.0 00000100:00080000:0.0:1767319815.113417:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319815.113423:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319815.300566:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319816.328312:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319817.112420:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319817.112496:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319817.112936:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646208 00000100:00100000:0.0:1767319817.112948:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646208 00000100:00100000:0.0:1767319817.113013:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646336 00000100:00100000:0.0:1767319817.113016:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646336 00000100:00100000:0.0:1767319817.113069:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1342 00000100:00100000:0.0:1767319817.113079:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915646336:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319817.113158:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915646336:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (663us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319817.113165:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1342 00000100:00100000:0.0:1767319817.113183:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1770 00000100:00100000:0.0:1767319817.113188:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915646208:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319817.113205:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319817.113230:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915646208:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (826us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319817.113236:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1770 00000100:00080000:2.0:1767319817.113641:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319817.353512:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319817.668887:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982911488:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319817.668930:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319817.668947:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982911616:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319817.668964:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319817.669555:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911488 00000100:00100000:2.0:1767319817.669564:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911488 00000100:00100000:2.0:1767319817.669635:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 906 00000100:00100000:2.0:1767319817.669645:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982911488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319817.669678:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319817.674112:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319817.674201:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911616 00000100:00100000:3.0:1767319817.674219:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911616 00000100:00100000:3.0:1767319817.674287:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 907 00000100:00100000:3.0:1767319817.674298:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982911616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319817.674334:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319817.674399:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982911616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 103us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319817.674407:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 907 00000100:00100000:3.0:1767319817.674472:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982911616:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319817.674513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319817.674525:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319817.677888:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982911488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 8235us (8915us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319817.677906:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 906 00000100:00100000:1.0:1767319817.683676:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982911488:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319818.372545:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319819.396486:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319820.228473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319820.228480:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319820.228549:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982911744:0@lo:400:kworker.0 00000100:00100000:0.0:1767319820.228617:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319820.228705:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911744 00000100:00100000:1.0:1767319820.228717:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911744 00000100:00080000:3.0:1767319820.228736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319820.228740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319820.228745:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319820.228750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5063 00000100:00080000:3.0:1767319820.228754:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319820.228760:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319820.228788:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3480 x1853163982911872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319820.228803:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1343 00000100:00100000:1.0:1767319820.228813:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982911744:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319820.228819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319820.228822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319820.228824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319820.228827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5063 00000100:00080000:3.0:1767319820.228829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319820.228833:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319820.228843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3b80 x1853163982912128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319820.228852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319820.228854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319820.228856:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319820.228859:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5063 00000100:00100000:2.0:1767319820.228897:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982912000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.228926:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319820.229025:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319820.229030:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:2.0:1767319820.229091:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319820.229108:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982911872:0@lo:38:kworker.0 00000100:00100000:2.0:1767319820.229134:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319820.229154:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982912128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319820.229171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319820.229198:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912000 00000100:00100000:2.0:1767319820.229202:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912000 00000100:00100000:3.0:1767319820.229453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1180 x1853163982912256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319820.229488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319820.229491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319820.229493:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319820.229497:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5063 00000100:00080000:3.0:1767319820.229500:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319820.229504:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319820.229520:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163982912640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319820.229540:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982912512:0@lo:400:kworker.0 00000100:00100000:3.0:1767319820.229574:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319820.229606:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982912256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319820.229630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319820.229647:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982912640:0@lo:38:kworker.0 00000100:00100000:3.0:1767319820.229664:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319820.229686:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912256 00000100:00100000:3.0:1767319820.229701:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912256 00000100:00100000:3.0:1767319820.229726:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 468 00000100:00100000:3.0:1767319820.229732:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982912256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319820.229752:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319820.229789:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982912256:12345-0@lo:38:kworker.0 Request processed in 55us (157us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319820.229797:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 468 00000100:00100000:3.0:1767319820.229821:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319820.229828:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982912384:0@lo:400:kworker.0 00000100:00100000:3.0:1767319820.229848:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319820.229870:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912512 00000100:00100000:3.0:1767319820.229874:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912512 00000100:00100000:3.0:1767319820.229906:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912384 00000100:00100000:3.0:1767319820.229909:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912384 00000100:00100000:3.0:1767319820.229928:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 835 00000100:00100000:3.0:1767319820.229934:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982912512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319820.229968:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982912512:12345-0@lo:400:kworker.0 Request processed in 34us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319820.229974:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 835 00000100:00100000:3.0:1767319820.229985:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 836 00000100:00100000:3.0:1767319820.229991:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982912384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319820.230020:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982912384:12345-0@lo:400:kworker.0 Request processed in 29us (170us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319820.230025:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 836 00000100:00100000:2.0:1767319820.230159:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982911872 00000100:00100000:2.0:1767319820.230164:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982911872 00000100:00100000:2.0:1767319820.230173:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912128 00000100:00100000:2.0:1767319820.230175:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912128 00000100:00100000:2.0:1767319820.230184:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912640 00000100:00100000:2.0:1767319820.230186:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912640 00000100:00100000:2.0:1767319820.230195:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1344 00000100:00100000:2.0:1767319820.230201:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982912000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.230233:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982912000:12345-0@lo:400:kworker.0 Request processed in 33us (1307us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319820.230239:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1344 00000100:00100000:2.0:1767319820.230251:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1771 00000100:00100000:2.0:1767319820.230255:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982911872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319820.230265:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319820.230301:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982911872:12345-0@lo:38:kworker.0 Request processed in 44us (1165us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319820.230307:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1771 00000100:00100000:2.0:1767319820.230317:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1772 00000100:00100000:2.0:1767319820.230320:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982912128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319820.230328:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319820.230349:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982912128:12345-0@lo:38:kworker.0 Request processed in 27us (1177us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319820.230354:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1772 00000100:00100000:2.0:1767319820.230362:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1773 00000100:00100000:2.0:1767319820.230366:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982912640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319820.230373:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319820.230391:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982912640:12345-0@lo:38:kworker.0 Request processed in 24us (725us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319820.230396:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1773 00000100:00100000:1.0:1767319820.230930:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982911744:12345-0@lo:400:kworker.0 Request processed in 2116us (2308us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319820.230938:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1343 00010000:00080000:0.0:1767319820.231441:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398ea00 x1853163982911744/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319836 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319820.231456:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982911744:0@lo:400:kworker.0 00000100:00080000:2.0:1767319820.232795:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319820.232806:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319820.232810:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319820.232863:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319820.232872:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319820.232876:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982912256:0@lo:38:kworker.0 00000100:00080000:2.0:1767319820.232890:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319820.232894:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319820.232896:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982912640:0@lo:38:kworker.0 00000100:00080000:2.0:1767319820.232909:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319820.232913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319820.232915:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982911872:0@lo:38:kworker.0 00000100:00080000:2.0:1767319820.232926:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319820.232930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319820.232933:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982912128:0@lo:38:kworker.0 00000100:00100000:2.0:1767319820.232999:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982912512:0@lo:400:kworker.0 00000100:00100000:2.0:1767319820.233146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982912384:0@lo:400:kworker.0 00010000:00080000:2.0:1767319820.233164:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e2300 x1853163982912000/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319836 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319820.233200:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982912000:0@lo:400:kworker.0 00010000:00010000:1.0:1767319820.420466:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319821.444827:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319822.222860:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319822.222906:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319822.223005:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646464 00000100:00100000:1.0:1767319822.223015:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646464 00000100:00100000:1.0:1767319822.223070:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646592 00000100:00100000:1.0:1767319822.223073:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646592 00000100:00100000:1.0:1767319822.223130:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1345 00000100:00100000:1.0:1767319822.223140:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915646592:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319822.223236:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915646592:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319822.223245:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1345 00000100:00100000:1.0:1767319822.223317:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1774 00000100:00100000:1.0:1767319822.223323:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915646464:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319822.223346:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319822.223379:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915646464:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (530us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319822.223386:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1774 00000100:00080000:2.0:1767319822.225853:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319822.468687:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319822.788518:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982912768:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319822.788561:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982912896:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319822.793638:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319822.794763:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319822.805871:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912896 00000100:00100000:2.0:1767319822.805890:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912896 00000100:00100000:2.0:1767319822.806005:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982912768 00000100:00100000:2.0:1767319822.806009:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982912768 00000100:00100000:2.0:1767319822.806027:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908 00000100:00100000:2.0:1767319822.806036:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982912896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319822.806077:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319822.806182:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982912896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 147us (12553us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319822.806190:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908 00000100:00100000:2.0:1767319822.806209:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909 00000100:00100000:2.0:1767319822.806215:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982912768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319822.806240:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319822.806286:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982912768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (11511us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319822.806293:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909 00000100:00100000:2.0:1767319822.806352:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982912896:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319822.808095:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319822.808104:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319822.808158:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982912768:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319823.492607:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319824.517013:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319825.348862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319825.348873:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319825.348978:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982913024:0@lo:400:kworker.0 00000100:00100000:2.0:1767319825.349052:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319825.349146:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913024 00000100:00100000:3.0:1767319825.349156:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913024 00000100:00100000:3.0:1767319825.349244:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1346 00000100:00100000:3.0:1767319825.349255:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982913024:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319825.349373:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319825.350698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319825.350707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319825.350711:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319825.350719:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5068 00000100:00080000:1.0:1767319825.350724:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319825.350732:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319825.350768:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982913152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319825.350810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319825.350813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319825.350815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319825.350818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5068 00000100:00080000:1.0:1767319825.350821:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319825.350825:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319825.350838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6300 x1853163982913408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319825.350846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319825.350849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319825.350850:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319825.350853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5068 00000100:00080000:1.0:1767319825.350855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319825.350859:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319825.350868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163982913536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319825.351680:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982913792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.351758:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.351826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982913664:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.351853:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.351882:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913792 00000100:00100000:0.0:1767319825.351890:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913792 00000100:00100000:0.0:1767319825.351949:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913664 00000100:00100000:0.0:1767319825.351952:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913664 00000100:00100000:0.0:1767319825.351979:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 837 00000100:00100000:0.0:1767319825.351988:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982913792:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352041:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982913792:12345-0@lo:400:kworker.0 Request processed in 55us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319825.352048:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 837 00000100:00100000:0.0:1767319825.352064:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 838 00000100:00100000:0.0:1767319825.352070:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982913664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352098:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982913664:12345-0@lo:400:kworker.0 Request processed in 28us (244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319825.352103:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 838 00000100:00080000:0.0:1767319825.352125:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319825.352130:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319825.352166:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982913792:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352195:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982913664:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352205:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319825.352210:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982913280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352254:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.352325:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913280 00000100:00100000:0.0:1767319825.352329:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913280 00000100:00100000:0.0:1767319825.352364:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347 00000100:00100000:0.0:1767319825.352370:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982913280:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352415:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982913280:12345-0@lo:400:kworker.0 Request processed in 45us (160us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319825.352421:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347 00000100:00080000:0.0:1767319825.352437:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319825.352456:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0380 x1853163982913280/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319841 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319825.352466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982913280:0@lo:400:kworker.0 00000100:00100000:0.0:1767319825.352511:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982913152:0@lo:38:kworker.0 00000100:00100000:0.0:1767319825.352537:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.352556:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982913408:0@lo:38:kworker.0 00000100:00100000:0.0:1767319825.352577:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.352605:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982913536:0@lo:38:kworker.0 00000100:00100000:0.0:1767319825.352629:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319825.352671:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913536 00000100:00100000:0.0:1767319825.352675:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913536 00000100:00100000:0.0:1767319825.352702:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 469 00000100:00100000:0.0:1767319825.352709:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982913536:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319825.352731:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319825.352767:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982913536:12345-0@lo:38:kworker.0 Request processed in 57us (137us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319825.352776:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 469 00000100:00100000:0.0:1767319825.352795:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913152 00000100:00100000:0.0:1767319825.352798:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913152 00000100:00100000:0.0:1767319825.352822:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913408 00000100:00100000:0.0:1767319825.352824:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913408 00000100:00100000:0.0:1767319825.352833:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1775 00000100:00100000:0.0:1767319825.352837:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982913152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319825.352847:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319825.352874:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982913152:12345-0@lo:38:kworker.0 Request processed in 36us (337us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319825.352879:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1775 00000100:00100000:0.0:1767319825.352889:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776 00000100:00100000:0.0:1767319825.352893:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982913408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319825.352901:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319825.352922:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982913408:12345-0@lo:38:kworker.0 Request processed in 27us (344us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319825.352928:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776 00000100:00080000:0.0:1767319825.352953:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319825.352958:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319825.352961:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982913152:0@lo:38:kworker.0 00000100:00080000:0.0:1767319825.352970:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319825.352974:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319825.352976:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982913408:0@lo:38:kworker.0 00000100:00080000:0.0:1767319825.352983:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319825.352986:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319825.352989:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982913536:0@lo:38:kworker.0 00000100:00100000:3.0:1767319825.358022:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982913024:12345-0@lo:400:kworker.0 Request processed in 8759us (8958us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319825.358038:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1346 00010000:00080000:2.0:1767319825.359801:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1880 x1853163982913024/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319841 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319825.359821:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982913024:0@lo:400:kworker.0 00000100:00080000:1.0:1767319825.361850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319825.361862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319825.361866:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319825.361875:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5068 00000100:00080000:1.0:1767319825.361880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319825.361887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319825.361934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982913920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319825.362026:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982913920:0@lo:38:kworker.0 00000100:00100000:0.0:1767319825.362098:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319825.364029:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982913920 00000100:00100000:3.0:1767319825.364043:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982913920 00000100:00100000:3.0:1767319825.364104:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1777 00000100:00100000:3.0:1767319825.364112:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982913920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319825.364135:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319825.368022:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982913920:12345-0@lo:38:kworker.0 Request processed in 3899us (5912us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319825.368040:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1777 00000100:00080000:0.0:1767319825.374732:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319825.374748:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319825.374754:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982913920:0@lo:38:kworker.0 00010000:00010000:2.0:1767319825.540890:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319826.564773:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319827.340953:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319827.340997:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319827.341085:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646720 00000100:00100000:2.0:1767319827.341096:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646720 00000100:00100000:2.0:1767319827.341143:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646848 00000100:00100000:2.0:1767319827.341146:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646848 00000100:00100000:2.0:1767319827.341207:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1348 00000100:00100000:2.0:1767319827.341217:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915646848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319827.341361:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915646848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 144us (362us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319827.341368:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1348 00000100:00100000:2.0:1767319827.341387:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1778 00000100:00100000:2.0:1767319827.341391:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915646720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319827.341409:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319827.341434:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915646720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (501us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319827.341440:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1778 00000100:00080000:3.0:1767319827.342261:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319827.588664:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319827.910674:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982914048:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319827.910772:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319827.911069:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914048 00000100:00100000:3.0:1767319827.911080:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914048 00000100:00100000:0.0:1767319827.911083:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319827.911094:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982914176:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319827.911123:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319827.911144:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914176 00000100:00100000:3.0:1767319827.911148:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914176 00000100:00100000:0.0:1767319827.911156:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 910 00000100:00100000:3.0:1767319827.911165:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 911 00000100:00100000:3.0:1767319827.911172:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982914176:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319827.911175:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982914048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319827.911202:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:3.0:1767319827.911202:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319827.911245:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982914176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (121us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319827.911254:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 911 00000100:00100000:3.0:1767319827.911331:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982914176:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319827.911479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319827.911485:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319827.911634:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982914048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 469us (861us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319827.911640:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 910 00000100:00100000:3.0:1767319827.911909:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982914048:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319828.612528:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319829.636454:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319830.468605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319830.468617:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319830.468680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319830.468684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319830.468689:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319830.468698:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5073 00000100:00080000:3.0:1767319830.468702:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319830.468708:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319830.468728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163982914432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319830.468792:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982914432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.468853:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319830.468935:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982914304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.468964:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319830.469005:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914432 00000100:00100000:0.0:1767319830.469012:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914432 00000100:00100000:0.0:1767319830.469055:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914304 00000100:00100000:0.0:1767319830.469058:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914304 00000100:00100000:0.0:1767319830.469094:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1349 00000100:00100000:0.0:1767319830.469103:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982914304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.469144:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982914304:12345-0@lo:400:kworker.0 Request processed in 42us (179us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319830.469155:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1349 00000100:00100000:0.0:1767319830.469169:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1779 00000100:00100000:0.0:1767319830.469173:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982914432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319830.469190:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319830.469298:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319830.469529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319830.469533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319830.469535:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319830.469540:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5073 00000100:00080000:3.0:1767319830.469543:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319830.469546:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319830.469558:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163982914688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319830.469600:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982914688:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.469624:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.469675:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982914560:0@lo:400:kworker.0 00000100:00100000:2.0:1767319830.469698:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319830.469722:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914688 00000100:00100000:2.0:1767319830.469726:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914688 00000100:00100000:2.0:1767319830.469744:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914560 00000100:00100000:2.0:1767319830.469747:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914560 00000100:00100000:2.0:1767319830.469767:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1350 00000100:00100000:2.0:1767319830.469774:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982914560:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319830.469804:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982914560:12345-0@lo:400:kworker.0 Request processed in 31us (105us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319830.469810:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1350 00000100:00100000:2.0:1767319830.469821:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780 00000100:00100000:2.0:1767319830.469824:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982914688:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319830.469834:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319830.469852:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982914688:12345-0@lo:38:kworker.0 Request processed in 26us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319830.469858:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780 00000100:00080000:2.0:1767319830.469870:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319830.469888:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35180 x1853163982914560/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319830.469900:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982914560:0@lo:400:kworker.0 00010000:00080000:2.0:1767319830.469949:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34380 x1853163982914304/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319846 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319830.469958:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982914304:0@lo:400:kworker.0 00000100:00100000:0.0:1767319830.470171:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982914432:12345-0@lo:38:kworker.0 Request processed in 996us (1317us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319830.470178:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1779 00000100:00080000:3.0:1767319830.470805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319830.470810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319830.470812:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319830.470817:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5073 00000100:00080000:3.0:1767319830.470820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319830.470824:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319830.470846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36680 x1853163982914816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319830.470899:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982914944:0@lo:400:kworker.0 00000100:00100000:2.0:1767319830.470934:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319830.470992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319830.470996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319830.470997:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319830.471001:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5073 00000100:00080000:3.0:1767319830.471003:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319830.471007:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319830.471016:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163982915200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319830.471049:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982915072:0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.471072:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319830.471199:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914944 00000100:00100000:0.0:1767319830.471205:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914944 00000100:00100000:0.0:1767319830.471259:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915072 00000100:00100000:0.0:1767319830.471262:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915072 00000100:00100000:0.0:1767319830.471303:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 839 00000100:00100000:0.0:1767319830.471310:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982914944:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319830.471375:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319830.471379:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319830.471400:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 840 00000100:00100000:3.0:1767319830.471408:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982915072:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319830.471441:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982915072:12345-0@lo:400:kworker.0 Request processed in 34us (368us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319830.471447:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 840 00000100:00100000:3.0:1767319830.471490:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982915072:0@lo:400:kworker.0 00000100:00080000:1.0:1767319830.471742:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319830.471747:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319830.471751:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982914432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.471766:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982914816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.471796:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319830.471832:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982915200:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.471852:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319830.471873:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319830.471877:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319830.471880:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982914688:0@lo:38:kworker.0 00000100:00100000:1.0:1767319830.471906:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915200 00000100:00100000:1.0:1767319830.471910:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915200 00000100:00100000:1.0:1767319830.471933:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1781 00000100:00100000:1.0:1767319830.471938:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982915200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319830.471954:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319830.471985:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982915200:12345-0@lo:38:kworker.0 Request processed in 45us (131us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319830.471990:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1781 00000100:00080000:1.0:1767319830.472013:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319830.472017:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319830.472019:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982915200:0@lo:38:kworker.0 00000100:00100000:3.0:1767319830.472202:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982914816 00000100:00100000:3.0:1767319830.472206:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982914816 00000100:00100000:3.0:1767319830.472244:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 470 00000100:00100000:3.0:1767319830.472250:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982914816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319830.472260:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319830.472306:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982914816:12345-0@lo:38:kworker.0 Request processed in 55us (509us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319830.472313:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 470 00000100:00100000:0.0:1767319830.472455:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982914944:12345-0@lo:400:kworker.0 Request processed in 1144us (1519us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319830.472462:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 839 00000100:00080000:1.0:1767319830.472626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319830.472630:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319830.472633:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982914816:0@lo:38:kworker.0 00000100:00100000:2.0:1767319830.472695:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982914944:0@lo:400:kworker.0 00010000:00010000:2.0:1767319830.663119:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319831.689848:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319832.460889:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319832.460944:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319832.461035:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647104 00000100:00100000:0.0:1767319832.461047:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647104 00000100:00100000:0.0:1767319832.461127:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915646976 00000100:00100000:0.0:1767319832.461131:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915646976 00000100:00100000:0.0:1767319832.461158:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1351 00000100:00100000:0.0:1767319832.461171:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915647104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319832.461192:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1782 00000100:00100000:1.0:1767319832.461201:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915646976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319832.461224:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319832.461254:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915647104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (377us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319832.461261:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915646976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (315us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319832.461268:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1351 00000100:00080000:3.0:1767319832.461535:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319832.469472:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1782 00010000:00010000:1.0:1767319832.708511:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319833.028917:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319833.028945:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982915328:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319833.029030:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319833.029186:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982915456:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319833.029223:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319833.029337:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915328 00000100:00100000:1.0:1767319833.029346:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915328 00000100:00100000:1.0:1767319833.029412:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915456 00000100:00100000:1.0:1767319833.029416:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915456 00000100:00100000:1.0:1767319833.029434:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912 00000100:00100000:1.0:1767319833.029443:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982915328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319833.029484:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319833.029539:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982915328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (508us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319833.029551:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912 00000100:00100000:1.0:1767319833.029572:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 913 00000100:00100000:1.0:1767319833.029579:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982915456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319833.029596:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319833.029625:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982915456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (401us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319833.029630:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 913 00000100:00080000:2.0:1767319833.029688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319833.029693:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319833.029732:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982915456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319833.029815:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982915328:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319833.732548:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319834.760281:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319835.592330:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.592343:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319835.592430:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.592435:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.592441:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.592452:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.592459:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.592468:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319835.592497:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982915712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319835.592540:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.592544:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.592546:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.592551:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.592553:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.592557:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319835.592568:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6d80 x1853163982915968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319835.592577:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.592581:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.592582:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.592585:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.592588:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.592592:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319835.592603:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982916096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319835.592624:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319835.592627:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319835.592629:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319835.592634:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5079 00000100:00080000:1.0:1767319835.592637:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319835.592641:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319835.592653:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5500 x1853163982916480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319835.592738:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982916224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.592836:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.592923:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319835.592930:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982915840:0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.592960:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.592994:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982915584:0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.593019:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.593044:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319835.593049:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982916352:0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.593070:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319835.593115:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915840 00000100:00100000:2.0:1767319835.593124:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915840 00000100:00100000:2.0:1767319835.593184:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915584 00000100:00100000:2.0:1767319835.593187:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915584 00000100:00100000:2.0:1767319835.593213:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352 00000100:00100000:2.0:1767319835.593223:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982915840:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.593315:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982915840:12345-0@lo:400:kworker.0 Request processed in 92us (352us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319835.593323:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352 00000100:00100000:2.0:1767319835.593339:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1353 00000100:00100000:2.0:1767319835.593345:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982915584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319835.593377:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982915584:12345-0@lo:400:kworker.0 Request processed in 32us (356us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319835.593383:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1353 00000100:00080000:2.0:1767319835.593417:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319835.593447:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6a00 x1853163982915584/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319835.593461:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982915584:0@lo:400:kworker.0 00010000:00080000:2.0:1767319835.593491:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5180 x1853163982915840/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319851 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319835.593499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982915840:0@lo:400:kworker.0 00000100:00100000:3.0:1767319835.594046:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916224 00000100:00100000:3.0:1767319835.594052:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916224 00000100:00100000:3.0:1767319835.594106:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916352 00000100:00100000:3.0:1767319835.594110:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916352 00000100:00100000:3.0:1767319835.594131:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 841 00000100:00100000:3.0:1767319835.594138:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982916224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319835.594186:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982916224:12345-0@lo:400:kworker.0 Request processed in 49us (1351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319835.594192:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 841 00000100:00100000:3.0:1767319835.594206:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 842 00000100:00100000:3.0:1767319835.594212:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982916352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319835.594239:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982916352:12345-0@lo:400:kworker.0 Request processed in 27us (1168us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319835.594245:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 842 00000100:00100000:3.0:1767319835.594308:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982916224:0@lo:400:kworker.0 00000100:00100000:0.0:1767319835.594532:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982915712:0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.594572:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.594614:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982915968:0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.594635:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.594653:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982916096:0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.594674:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.594697:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982916480:0@lo:38:kworker.0 00000100:00100000:0.0:1767319835.594717:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319835.594745:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916096 00000100:00100000:0.0:1767319835.594749:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916096 00000100:00100000:0.0:1767319835.594790:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 471 00000100:00100000:0.0:1767319835.594795:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982916096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.594814:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.594849:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982916096:12345-0@lo:38:kworker.0 Request processed in 52us (174us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319835.594856:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 471 00000100:00100000:0.0:1767319835.594873:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915712 00000100:00100000:0.0:1767319835.594877:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915712 00000100:00100000:0.0:1767319835.594894:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982915968 00000100:00100000:0.0:1767319835.594897:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982915968 00000100:00100000:0.0:1767319835.594907:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916480 00000100:00100000:0.0:1767319835.594909:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916480 00000100:00100000:0.0:1767319835.594918:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1783 00000100:00100000:0.0:1767319835.594923:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982915712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.594933:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.594958:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982915712:12345-0@lo:38:kworker.0 Request processed in 34us (385us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319835.594964:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1783 00000100:00100000:0.0:1767319835.594974:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1784 00000100:00100000:0.0:1767319835.594978:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982915968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.594987:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.595007:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982915968:12345-0@lo:38:kworker.0 Request processed in 27us (371us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319835.595011:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1784 00000100:00100000:0.0:1767319835.595020:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1785 00000100:00100000:0.0:1767319835.595023:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982916480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319835.595030:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319835.595049:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982916480:12345-0@lo:38:kworker.0 Request processed in 24us (331us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319835.595055:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1785 00000100:00080000:0.0:1767319835.595099:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.595102:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.595105:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982915712:0@lo:38:kworker.0 00000100:00080000:0.0:1767319835.595111:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.595113:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.595114:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982915968:0@lo:38:kworker.0 00000100:00080000:0.0:1767319835.595119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.595121:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.595159:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982916096:0@lo:38:kworker.0 00000100:00080000:0.0:1767319835.595164:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319835.595166:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319835.595167:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982916480:0@lo:38:kworker.0 00000100:00080000:2.0:1767319835.595736:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319835.595742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319835.595788:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982916352:0@lo:400:kworker.0 00010000:00010000:1.0:1767319835.780497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319836.806406:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767319837.587477:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915647232 offset 0 length 520: 4 00000100:00100000:3.0:1767319837.587668:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319837.587757:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319837.593737:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647360 00000100:00100000:2.0:1767319837.593759:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647360 00000100:00100000:2.0:1767319837.593824:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647488 00000100:00100000:2.0:1767319837.593828:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647488 00000100:00100000:2.0:1767319837.593899:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1354 00000100:00100000:2.0:1767319837.593911:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915647488:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319837.594003:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915647488:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (6244us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319837.594011:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1354 00000100:00100000:2.0:1767319837.594032:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1786 00000100:00100000:2.0:1767319837.594036:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915647360:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319837.594059:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319837.594087:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915647360:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (6420us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319837.594093:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1786 00000100:00080000:3.0:1767319837.594283:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319837.840888:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319838.148875:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319838.150305:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982916736:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319838.150407:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319838.151669:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916736 00000100:00100000:0.0:1767319838.151686:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916736 00000100:00100000:0.0:1767319838.151795:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 914 00000100:00100000:0.0:1767319838.151808:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982916736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319838.151844:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319838.151920:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982916736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 115us (1513us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319838.151928:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 914 00000100:00100000:0.0:1767319838.151990:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982916736:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319838.152223:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319838.158882:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982916608:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319838.158988:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319838.159126:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916608 00000100:00100000:1.0:1767319838.159135:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916608 00000100:00100000:1.0:1767319838.159192:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915 00000100:00100000:1.0:1767319838.159201:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982916608:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319838.159237:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767319838.159286:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319838.159307:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982916608:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319838.159315:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915 00000100:00100000:3.0:1767319838.159576:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982916608:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319838.852502:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319839.879450:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319840.708929:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.708940:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319840.708998:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.709002:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319840.709005:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319840.709014:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5084 00000100:00080000:1.0:1767319840.709018:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319840.709024:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319840.709056:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4380 x1853163982916992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319840.709099:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.709102:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319840.709104:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319840.709108:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5084 00000100:00080000:1.0:1767319840.709111:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319840.709115:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319840.709129:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982917248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319840.709138:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.709141:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319840.709143:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319840.709147:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5084 00000100:00080000:1.0:1767319840.709149:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319840.709153:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319840.709164:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163982917376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319840.709186:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.709189:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319840.709191:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319840.709194:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5084 00000100:00080000:1.0:1767319840.709196:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319840.709199:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319840.709209:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6d80 x1853163982917760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319840.709334:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319815/real 1767319815] req@ffff93463396c380 x1853163982910336/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319840.709369:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463396c380 x1853163982910336/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319840.709387:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982916992:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.709450:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.709511:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982917248:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.709533:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.709551:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982917376:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.709574:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.709596:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982917760:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.709615:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319840.709632:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319840.709642:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319840.709645:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982910336:0@lo:250:kworker.0 00000100:00100000:3.0:1767319840.709675:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982917120:0@lo:400:kworker.0 00000100:00100000:3.0:1767319840.709756:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.710102:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319840.710110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982917504:0@lo:400:kworker.0 00000100:00100000:3.0:1767319840.710138:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.710172:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916992 00000100:00100000:3.0:1767319840.710180:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916992 00000100:00100000:3.0:1767319840.710207:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917248 00000100:00100000:3.0:1767319840.710211:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917248 00000100:00100000:3.0:1767319840.710218:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917760 00000100:00100000:3.0:1767319840.710220:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917760 00000100:00100000:3.0:1767319840.710226:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917120 00000100:00100000:3.0:1767319840.710228:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917120 00000100:00100000:3.0:1767319840.710257:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1355 00000100:00100000:3.0:1767319840.710266:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982917120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710402:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982916864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710434:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319840.710451:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319840.710457:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982917632:0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710479:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319840.710500:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982917120:12345-0@lo:400:kworker.0 Request processed in 233us (741us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319840.710511:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1355 00000100:00100000:2.0:1767319840.710517:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917504 00000100:00100000:2.0:1767319840.710522:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917504 00000100:00100000:3.0:1767319840.710533:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982916864 00000100:00100000:3.0:1767319840.710538:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982916864 00000100:00100000:3.0:1767319840.710565:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1356 00000100:00100000:3.0:1767319840.710573:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982916864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710577:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917632 00000100:00100000:2.0:1767319840.710582:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917632 00000100:00100000:2.0:1767319840.710609:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 843 00000100:00100000:3.0:1767319840.710615:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982916864:12345-0@lo:400:kworker.0 Request processed in 43us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319840.710620:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982917504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319840.710628:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1356 00000100:00100000:3.0:1767319840.710645:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1787 00000100:00100000:3.0:1767319840.710650:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982916992:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319840.710661:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982917504:12345-0@lo:400:kworker.0 Request processed in 43us (520us total) trans 0 rc 0/0 00010000:00020000:3.0:1767319840.710670:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319840.710671:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 843 00000100:00100000:2.0:1767319840.710688:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 844 00000100:00100000:2.0:1767319840.710695:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982917632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710717:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982917632:12345-0@lo:400:kworker.0 Request processed in 23us (237us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319840.710726:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 844 00000100:00080000:2.0:1767319840.710748:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319840.710751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319840.710753:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319840.710778:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982917632:0@lo:400:kworker.0 00010000:00080000:2.0:1767319840.710789:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6300 x1853163982916864/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319840.710799:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982916864:0@lo:400:kworker.0 00000100:00100000:2.0:1767319840.710829:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788 00000100:00100000:2.0:1767319840.710833:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982917248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319840.710844:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319840.710875:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982917248:12345-0@lo:38:kworker.0 Request processed in 40us (1341us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319840.710881:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788 00000100:00100000:2.0:1767319840.710894:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1789 00000100:00100000:2.0:1767319840.710899:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982917760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319840.710909:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319840.710917:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982916992:12345-0@lo:38:kworker.0 Request processed in 263us (1464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319840.710925:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1787 00000100:00100000:2.0:1767319840.710939:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982917760:12345-0@lo:38:kworker.0 Request processed in 38us (1321us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319840.710947:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1789 00000100:00100000:3.0:1767319840.710950:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982917376 00000100:00100000:3.0:1767319840.710955:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982917376 00000100:00100000:3.0:1767319840.710977:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 472 00000100:00080000:2.0:1767319840.710979:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319840.710986:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982917376:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767319840.710990:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319840.710996:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982916992:0@lo:38:kworker.0 00010000:00020000:3.0:1767319840.711000:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767319840.711022:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319840.711028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319840.711032:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982917248:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.711038:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982917376:12345-0@lo:38:kworker.0 Request processed in 52us (1459us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319840.711046:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 472 00000100:00080000:2.0:1767319840.711047:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319840.711052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319840.711057:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982917376:0@lo:38:kworker.0 00000100:00080000:2.0:1767319840.711068:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319840.711072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319840.711075:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982917760:0@lo:38:kworker.0 00000100:00100000:3.0:1767319840.711106:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982917504:0@lo:400:kworker.0 00010000:00080000:3.0:1767319840.711119:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7480 x1853163982917120/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319856 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319840.711129:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982917120:0@lo:400:kworker.0 00000100:00080000:1.0:1767319840.711182:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319840.711187:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319840.711189:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319840.711194:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5063 00000100:00080000:1.0:1767319840.711198:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319840.711203:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319840.711222:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982917888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319840.711302:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982917888:0@lo:250:kworker.0 00000400:00000100:2.0:1767319840.711335:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982917888 offset 0 length 520: 4 00010000:00010000:2.0:1767319840.900750:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319841.933308:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319842.703759:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319842.703864:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319842.703865:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647744 00000100:00100000:1.0:1767319842.703876:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647744 00000100:00080000:2.0:1767319842.703914:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319842.703932:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647616 00000100:00100000:1.0:1767319842.703936:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647616 00000100:00100000:2.0:1767319842.703957:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1357 00000100:00100000:2.0:1767319842.703969:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915647744:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319842.704041:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915647744:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (293us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319842.705852:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1790 00000100:00100000:1.0:1767319842.705862:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915647616:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319842.705884:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319842.706051:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915647616:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 188us (2186us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319842.706061:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1790 00000100:00100000:2.0:1767319842.708062:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1357 00010000:00010000:2.0:1767319842.948499:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319843.268565:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319843.268591:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982918016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319843.268768:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982918144:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319843.273026:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319843.273210:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319843.273316:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918144 00000100:00100000:2.0:1767319843.273327:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918144 00000100:00100000:2.0:1767319843.273395:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918016 00000100:00100000:2.0:1767319843.273399:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918016 00000100:00100000:2.0:1767319843.273418:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916 00000100:00100000:2.0:1767319843.273427:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982918144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319843.273458:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319843.273510:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982918144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (490us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319843.273517:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916 00000100:00100000:2.0:1767319843.273534:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917 00000100:00100000:2.0:1767319843.273540:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982918016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319843.273555:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319843.273587:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982918016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (378us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319843.273594:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917 00000100:00100000:2.0:1767319843.273637:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982918016:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319843.273668:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982918144:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319843.276314:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319843.276327:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319843.974136:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319844.996529:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319845.829651:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319845.829660:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319845.829706:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319845.829710:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319845.829714:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319845.829723:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319845.829729:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319845.829737:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319845.829742:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319845.829761:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247b80 x1853163982918400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319845.829761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982918272:0@lo:400:kworker.0 00000100:00100000:3.0:1767319845.829841:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319845.829859:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982918400:0@lo:38:kworker.0 00000100:00100000:3.0:1767319845.829908:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918272 00000100:00100000:1.0:1767319845.829909:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319845.829917:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918272 00000100:00080000:2.0:1767319845.829974:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319845.829977:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319845.829979:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319845.829979:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918400 00000100:00080000:2.0:1767319845.829983:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5089 00000100:00100000:3.0:1767319845.829985:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918400 00000100:00080000:2.0:1767319845.829986:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319845.829991:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319845.830002:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2300 x1853163982918656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319845.830012:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1358 00000100:00100000:3.0:1767319845.830022:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982918272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319845.830075:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982918272:12345-0@lo:400:kworker.0 Request processed in 54us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319845.830077:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1791 00000100:00100000:3.0:1767319845.830082:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1358 00000100:00100000:0.0:1767319845.830083:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982918400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319845.830101:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767319845.830102:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319845.830132:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e246680 x1853163982918272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319861 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319845.830145:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982918272:0@lo:400:kworker.0 00000100:00100000:1.0:1767319845.831744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982918656:0@lo:38:kworker.0 00000100:00100000:1.0:1767319845.831803:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319845.832482:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319845.832489:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319845.832492:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319845.832497:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319845.832501:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319845.832508:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319845.832539:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1c00 x1853163982918784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319845.832796:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319845.832800:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319845.832802:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319845.832806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5089 00000100:00080000:2.0:1767319845.832808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319845.832812:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319845.832824:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1f80 x1853163982919168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319845.832876:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982918912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.832928:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319845.832969:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319845.832976:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982918528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319845.833034:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918912 00000100:00100000:2.0:1767319845.833041:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918912 00000100:00100000:2.0:1767319845.833092:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 845 00000100:00100000:2.0:1767319845.833100:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982918912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833161:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982918912:12345-0@lo:400:kworker.0 Request processed in 49us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319845.833169:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 845 00000100:00080000:2.0:1767319845.833192:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319845.833230:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982918912:0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833255:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918656 00000100:00100000:2.0:1767319845.833259:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918656 00000100:00100000:2.0:1767319845.833297:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918528 00000100:00100000:2.0:1767319845.833300:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918528 00000100:00100000:2.0:1767319845.833324:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1359 00000100:00100000:2.0:1767319845.833330:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982918528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833611:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982918528:12345-0@lo:400:kworker.0 Request processed in 279us (608us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319845.833620:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1359 00000100:00100000:2.0:1767319845.833639:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792 00000100:00100000:2.0:1767319845.833644:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982918656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319845.833665:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319845.833707:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982918656:12345-0@lo:38:kworker.0 Request processed in 62us (1904us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319845.833712:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792 00000100:00080000:2.0:1767319845.833726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319845.833744:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e244000 x1853163982918528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319861 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319845.833755:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982918528:0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833777:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319845.833786:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982919040:0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833817:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319845.833854:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919040 00000100:00100000:2.0:1767319845.833859:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919040 00000100:00100000:2.0:1767319845.833893:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 846 00000100:00100000:2.0:1767319845.833901:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982919040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319845.833935:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982919040:12345-0@lo:400:kworker.0 Request processed in 35us (117us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319845.833941:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 846 00000100:00080000:3.0:1767319845.836581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319845.836631:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982919040:0@lo:400:kworker.0 00000100:00100000:0.0:1767319845.843048:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982918400:12345-0@lo:38:kworker.0 Request processed in 12951us (13126us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319845.843063:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1791 00000100:00080000:1.0:1767319845.845319:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319845.845332:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319845.845337:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982918400:0@lo:38:kworker.0 00000100:00100000:1.0:1767319845.845388:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982918784:0@lo:38:kworker.0 00000100:00100000:1.0:1767319845.845459:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319845.845488:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982919168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319845.845514:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319845.845554:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319845.845569:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319845.845572:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982918656:0@lo:38:kworker.0 00000100:00100000:0.0:1767319845.846232:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982918784 00000100:00100000:0.0:1767319845.846241:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982918784 00000100:00100000:0.0:1767319845.846305:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 473 00000100:00100000:0.0:1767319845.846313:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982918784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319845.846335:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319845.846382:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982918784:12345-0@lo:38:kworker.0 Request processed in 69us (922us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319845.846389:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 473 00000100:00080000:1.0:1767319845.847080:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319845.847088:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319845.847092:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982918784:0@lo:38:kworker.0 00000100:00100000:3.0:1767319845.849671:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919168 00000100:00100000:3.0:1767319845.849687:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919168 00000100:00100000:3.0:1767319845.849760:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1793 00000100:00100000:3.0:1767319845.849768:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982919168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319845.849793:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319845.849888:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982919168:12345-0@lo:38:kworker.0 Request processed in 119us (4372us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319845.849895:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1793 00000100:00080000:1.0:1767319845.850159:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319845.850166:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319845.850170:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982919168:0@lo:38:kworker.0 00010000:00010000:2.0:1767319846.020511:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319847.046076:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319847.823457:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319847.823767:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915647872 00000100:00100000:0.0:1767319847.823776:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915647872 00000100:00100000:0.0:1767319847.823828:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1794 00000100:00100000:0.0:1767319847.823837:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915647872:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319847.823858:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319847.823929:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915647872:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (483us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319847.823936:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1794 00000100:00100000:0.0:1767319847.833002:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319847.833104:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648000 00000100:00100000:3.0:1767319847.833123:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648000 00000100:00100000:3.0:1767319847.833232:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1360 00000100:00100000:3.0:1767319847.833250:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915648000:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319847.833374:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915648000:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 133us (382us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319847.833391:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1360 00000100:00080000:2.0:1767319847.834800:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319848.068711:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319848.388806:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319848.389353:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319848.389383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982919424:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319848.389730:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319848.389841:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919424 00000100:00100000:0.0:1767319848.389849:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919424 00000100:00100000:0.0:1767319848.389917:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 918 00000100:00100000:0.0:1767319848.389928:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982919424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319848.389960:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319848.390016:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982919424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319848.390024:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 918 00000100:00080000:2.0:1767319848.390082:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319848.390328:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982919424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319848.390747:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982919296:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319848.390860:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319848.390971:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919296 00000100:00100000:0.0:1767319848.390977:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919296 00000100:00100000:0.0:1767319848.391066:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 919 00000100:00100000:0.0:1767319848.391074:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982919296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319848.391098:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319848.391145:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982919296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319848.391153:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 919 00000100:00100000:1.0:1767319848.391244:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982919296:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319848.391687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767319849.095138:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319850.120263:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319850.948720:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319850.948730:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319850.948833:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982919552:0@lo:400:kworker.0 00000100:00100000:3.0:1767319850.949176:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319850.949325:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319850.949330:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319850.949334:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319850.949339:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5094 00000100:00080000:0.0:1767319850.949342:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319850.949349:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319850.949371:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982919680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319850.949404:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319850.949407:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319850.949410:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319850.949414:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5094 00000100:00080000:0.0:1767319850.949416:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319850.949420:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319850.949434:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d880 x1853163982919936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319850.949443:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319850.949446:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319850.949448:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319850.949450:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5094 00000100:00080000:0.0:1767319850.949452:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319850.949456:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319850.949469:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ea00 x1853163982920064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319850.949606:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982919808:0@lo:400:kworker.0 00000100:00100000:1.0:1767319850.949642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319850.951509:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319850.951525:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982920192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319850.951581:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319850.951630:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919552 00000100:00100000:1.0:1767319850.951640:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919552 00000100:00100000:1.0:1767319850.951696:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919808 00000100:00100000:1.0:1767319850.951700:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919808 00000100:00100000:1.0:1767319850.951726:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1361 00000100:00100000:1.0:1767319850.951735:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982919552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319850.951823:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1362 00000100:00100000:2.0:1767319850.951833:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982919808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319850.952192:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982919808:12345-0@lo:400:kworker.0 Request processed in 360us (2548us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319850.952201:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1362 00000100:00100000:2.0:1767319850.952230:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920192 00000100:00100000:2.0:1767319850.952234:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920192 00000100:00080000:0.0:1767319850.952516:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319850.952522:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319850.952526:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319850.952531:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5094 00000100:00080000:0.0:1767319850.952535:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319850.952539:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319850.952568:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163982920448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319850.952634:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982920320:0@lo:400:kworker.0 00000100:00100000:0.0:1767319850.952670:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319850.952706:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920320 00000100:00100000:0.0:1767319850.952710:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920320 00000100:00100000:0.0:1767319850.952738:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 847 00000100:00100000:0.0:1767319850.952744:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982920192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319850.952778:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982920192:12345-0@lo:400:kworker.0 Request processed in 33us (1197us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319850.952784:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 847 00000100:00100000:0.0:1767319850.952796:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 848 00000100:00100000:0.0:1767319850.952802:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982920320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319850.952835:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982920320:12345-0@lo:400:kworker.0 Request processed in 33us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319850.952840:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 848 00000100:00100000:1.0:1767319850.953540:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982919552:12345-0@lo:400:kworker.0 Request processed in 1802us (4360us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319850.953548:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1361 00000100:00080000:2.0:1767319850.953662:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319850.953667:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319850.953670:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319850.953700:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982920320:0@lo:400:kworker.0 00010000:00080000:3.0:1767319850.954257:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1880 x1853163982919552/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319866 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319850.954294:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982919552:0@lo:400:kworker.0 00000100:00100000:0.0:1767319850.954735:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982919680:0@lo:38:kworker.0 00000100:00100000:0.0:1767319850.954795:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319850.954823:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982919936:0@lo:38:kworker.0 00000100:00100000:0.0:1767319850.954845:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319850.954888:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982920064:0@lo:38:kworker.0 00000100:00100000:0.0:1767319850.954927:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319850.955008:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919680 00000100:00100000:2.0:1767319850.955016:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919680 00000100:00100000:2.0:1767319850.955045:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982919936 00000100:00100000:2.0:1767319850.955047:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982919936 00000100:00100000:2.0:1767319850.955074:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1795 00000100:00100000:2.0:1767319850.955080:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982919680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319850.955101:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319850.955150:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982919680:12345-0@lo:38:kworker.0 Request processed in 68us (354us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319850.955156:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1795 00000100:00100000:2.0:1767319850.955171:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1796 00000100:00100000:2.0:1767319850.955174:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982919936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319850.955183:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319850.955205:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982919936:12345-0@lo:38:kworker.0 Request processed in 29us (360us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319850.955211:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1796 00000100:00100000:2.0:1767319850.955249:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920064 00000100:00100000:2.0:1767319850.955658:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920064 00000100:00100000:2.0:1767319850.955684:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 474 00000100:00100000:2.0:1767319850.955689:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638757c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982920064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319850.955700:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319850.955730:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638757c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982920064:12345-0@lo:38:kworker.0 Request processed in 41us (802us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319850.955737:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 474 00000100:00080000:0.0:1767319850.956627:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319850.956634:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319850.956638:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982920064:0@lo:38:kworker.0 00000100:00100000:0.0:1767319850.956647:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982920448:0@lo:38:kworker.0 00000100:00100000:0.0:1767319850.956685:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319850.956735:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319850.956739:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319850.956742:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982919680:0@lo:38:kworker.0 00000100:00080000:0.0:1767319850.956752:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319850.956755:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319850.956758:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982919936:0@lo:38:kworker.0 00000100:00100000:1.0:1767319850.956820:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920448 00000100:00100000:1.0:1767319850.956826:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920448 00000100:00100000:1.0:1767319850.956851:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1797 00000100:00100000:1.0:1767319850.956866:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982920448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319850.956881:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319850.956931:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982920448:12345-0@lo:38:kworker.0 Request processed in 73us (245us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319850.956937:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1797 00000100:00080000:0.0:1767319850.957518:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319850.957524:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319850.957527:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982920448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319850.957616:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982920192:0@lo:400:kworker.0 00010000:00080000:2.0:1767319850.957632:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543e680 x1853163982919808/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319866 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319850.957642:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982919808:0@lo:400:kworker.0 00010000:00010000:1.0:1767319851.140731:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319852.164993:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319852.940837:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319852.940949:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319852.944157:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648256 00000100:00100000:3.0:1767319852.944172:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648256 00000100:00100000:3.0:1767319852.944256:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648128 00000100:00100000:3.0:1767319852.944260:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648128 00000100:00100000:3.0:1767319852.944308:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1363 00000100:00100000:3.0:1767319852.944319:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915648256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319852.944412:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915648256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (3584us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319852.944420:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1363 00000100:00100000:3.0:1767319852.944450:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1798 00000100:00100000:3.0:1767319852.944455:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915648128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319852.944476:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319852.944502:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915648128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (3554us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319852.944508:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1798 00000100:00080000:0.0:1767319852.944856:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319853.189012:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319853.510749:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319853.510774:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982920576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319853.510844:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319853.511203:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920576 00000100:00100000:3.0:1767319853.511212:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920576 00000100:00100000:3.0:1767319853.511298:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920 00000100:00100000:3.0:1767319853.511308:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982920576:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319853.511319:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319853.511330:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982920704:0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319853.511339:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319853.511360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319853.511400:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982920576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (554us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319853.511430:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982920576:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319853.511460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319853.511572:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920704 00000100:00100000:2.0:1767319853.511577:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920704 00000100:00080000:1.0:1767319853.511627:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319853.512135:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920 00000100:00100000:2.0:1767319853.516831:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921 00000100:00100000:2.0:1767319853.516858:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982920704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319853.516897:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319853.517021:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982920704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 165us (5648us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319853.517037:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921 00000100:00100000:1.0:1767319853.517146:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982920704:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319854.215647:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319855.236578:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319856.068503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319856.068514:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319856.068620:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982920832:0@lo:400:kworker.0 00000100:00100000:2.0:1767319856.068693:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319856.068829:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920832 00000100:00100000:3.0:1767319856.068838:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920832 00000100:00100000:3.0:1767319856.068896:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1364 00000100:00100000:3.0:1767319856.068906:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982920832:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767319856.069012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319856.069172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319856.069176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319856.069180:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319856.069186:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5099 00000100:00080000:1.0:1767319856.069190:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319856.069196:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319856.069224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982920960/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319856.069259:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319856.069263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319856.069265:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319856.069268:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5099 00000100:00080000:1.0:1767319856.069292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319856.069296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319856.069307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57b80 x1853163982921216/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319856.069316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319856.069319:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319856.069321:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319856.069324:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5099 00000100:00080000:1.0:1767319856.069326:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319856.069330:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319856.069340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55880 x1853163982921344/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319856.069399:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982921600:0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069435:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.069472:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921600 00000100:00100000:0.0:1767319856.069476:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921600 00000100:00100000:0.0:1767319856.069513:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 849 00000100:00100000:0.0:1767319856.069522:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982921600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069563:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982921600:12345-0@lo:400:kworker.0 Request processed in 43us (127us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319856.069570:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 849 00000100:00080000:0.0:1767319856.069592:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319856.069618:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982921600:0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069648:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982921088:0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069674:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.069691:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319856.069696:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982921472:0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069716:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.069733:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921472 00000100:00100000:0.0:1767319856.069736:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921472 00000100:00100000:0.0:1767319856.069775:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 850 00000100:00100000:0.0:1767319856.069780:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982921472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069808:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982921472:12345-0@lo:400:kworker.0 Request processed in 28us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319856.069813:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 850 00000100:00100000:0.0:1767319856.069826:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921088 00000100:00100000:0.0:1767319856.069875:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921088 00000100:00100000:0.0:1767319856.069899:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1365 00000100:00100000:0.0:1767319856.069904:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982921088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319856.069929:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982921088:12345-0@lo:400:kworker.0 Request processed in 25us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319856.069934:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1365 00000100:00080000:0.0:1767319856.069946:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319856.069948:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319856.069956:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982920960:0@lo:38:kworker.0 00000100:00100000:0.0:1767319856.069975:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.069997:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982921216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319856.070013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.070037:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982921344:0@lo:38:kworker.0 00000100:00100000:0.0:1767319856.070057:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319856.070085:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982920960 00000100:00100000:0.0:1767319856.070088:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982920960 00000100:00100000:0.0:1767319856.070106:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921216 00000100:00100000:0.0:1767319856.070108:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921216 00000100:00100000:0.0:1767319856.070118:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1799 00000100:00100000:0.0:1767319856.070121:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982920960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319856.070139:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319856.070166:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982920960:12345-0@lo:38:kworker.0 Request processed in 43us (190us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319856.070172:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1799 00000100:00100000:0.0:1767319856.070182:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800 00000100:00100000:0.0:1767319856.070185:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982921216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319856.070192:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319856.070208:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982921216:12345-0@lo:38:kworker.0 Request processed in 22us (194us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319856.070212:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800 00000100:00100000:0.0:1767319856.070235:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982921472:0@lo:400:kworker.0 00010000:00080000:0.0:1767319856.070244:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57800 x1853163982921088/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319872 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319856.070252:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982921088:0@lo:400:kworker.0 00000100:00080000:0.0:1767319856.070305:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319856.070310:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319856.070313:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982920960:0@lo:38:kworker.0 00000100:00080000:0.0:1767319856.070322:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319856.070324:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319856.070327:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982921216:0@lo:38:kworker.0 00000100:00100000:0.0:1767319856.070361:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921344 00000100:00100000:0.0:1767319856.070364:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921344 00000100:00100000:0.0:1767319856.070383:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 475 00000100:00100000:0.0:1767319856.070388:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982921344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319856.070399:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319856.070425:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982921344:12345-0@lo:38:kworker.0 Request processed in 36us (368us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319856.070432:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 475 00000100:00080000:0.0:1767319856.070454:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319856.070458:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319856.070461:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982921344:0@lo:38:kworker.0 00000100:00100000:3.0:1767319856.071002:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982920832:12345-0@lo:400:kworker.0 Request processed in 2094us (2305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319856.071010:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1364 00000100:00080000:1.0:1767319856.072649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319856.072658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319856.072663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319856.072670:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5099 00000100:00080000:1.0:1767319856.072674:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319856.072680:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319856.072719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56680 x1853163982921728/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319856.072904:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982921728:0@lo:38:kworker.0 00000100:00100000:0.0:1767319856.072968:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767319856.073308:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7480 x1853163982920832/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319872 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319856.073328:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982920832:0@lo:400:kworker.0 00000100:00100000:3.0:1767319856.073428:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921728 00000100:00100000:3.0:1767319856.073435:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921728 00000100:00100000:3.0:1767319856.073514:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1801 00000100:00100000:3.0:1767319856.073522:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982921728:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319856.073540:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319856.073594:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982921728:12345-0@lo:38:kworker.0 Request processed in 72us (625us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319856.073600:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1801 00000100:00080000:1.0:1767319856.073656:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319856.073661:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319856.073664:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982921728:0@lo:38:kworker.0 00010000:00010000:2.0:1767319856.260846:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319857.285707:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319858.059631:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319858.059694:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648384 00000100:00100000:3.0:1767319858.059706:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648384 00000100:00100000:3.0:1767319858.059766:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1802 00000100:00100000:3.0:1767319858.059835:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915648384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319858.059860:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319858.059974:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915648384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 140us (362us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319858.059982:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1802 00000100:00100000:1.0:1767319858.062210:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319858.062661:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648512 00000100:00100000:3.0:1767319858.062672:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648512 00000100:00100000:3.0:1767319858.062762:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1366 00000100:00100000:3.0:1767319858.062772:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915648512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319858.062860:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915648512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (660us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319858.062868:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1366 00000100:00080000:1.0:1767319858.063131:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319858.309058:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319858.628673:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982921856:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319858.628753:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319858.628776:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982921984:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319858.628814:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319858.628888:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921856 00000100:00100000:3.0:1767319858.628897:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921856 00000100:00100000:3.0:1767319858.628965:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982921984 00000100:00100000:3.0:1767319858.628969:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982921984 00000100:00100000:3.0:1767319858.629003:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 922 00000100:00100000:3.0:1767319858.629012:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982921856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319858.629054:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319858.629117:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982921856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 106us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319858.629125:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 922 00000100:00100000:3.0:1767319858.629144:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923 00000100:00100000:3.0:1767319858.629149:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982921984:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319858.629166:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319858.629238:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982921856:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319858.629329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319858.629333:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319858.631218:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982921984:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2058us (2393us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319858.631232:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923 00000100:00100000:1.0:1767319858.631294:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982921984:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319859.335352:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319860.358148:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319861.188523:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319861.188534:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319861.188601:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319861.188606:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319861.188610:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319861.188618:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5104 00000100:00080000:0.0:1767319861.188623:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319861.188629:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319861.188653:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c380 x1853163982922240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319861.188699:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319861.188703:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319861.188705:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319861.188709:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5104 00000100:00080000:0.0:1767319861.188712:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319861.188716:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319861.188728:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ed80 x1853163982922496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319861.188737:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319861.188740:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319861.188742:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319861.188745:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5104 00000100:00080000:0.0:1767319861.188748:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319861.188764:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319861.188778:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163982922624/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319861.188798:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319861.188801:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319861.188803:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319861.188806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5104 00000100:00080000:0.0:1767319861.188808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319861.188812:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319861.188824:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911180 x1853163982923008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319861.188910:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982922752:0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.188993:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189077:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319861.189083:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982922368:0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.189109:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189143:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922368 00000100:00100000:3.0:1767319861.189151:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922368 00000100:00100000:3.0:1767319861.189205:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1367 00000100:00100000:3.0:1767319861.189215:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982922368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319861.189268:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982922368:12345-0@lo:400:kworker.0 Request processed in 54us (157us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319861.189307:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1367 00000100:00080000:3.0:1767319861.189329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319861.189341:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982922240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319861.189365:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189386:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982922496:0@lo:38:kworker.0 00000100:00100000:3.0:1767319861.189407:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189421:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982922624:0@lo:38:kworker.0 00000100:00100000:3.0:1767319861.189441:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189463:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982923008:0@lo:38:kworker.0 00000100:00100000:3.0:1767319861.189481:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319861.189510:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922624 00000100:00100000:3.0:1767319861.189514:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922624 00000100:00100000:3.0:1767319861.189533:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 476 00000100:00100000:3.0:1767319861.189538:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982922624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319861.189559:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319861.189594:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982922624:12345-0@lo:38:kworker.0 Request processed in 54us (151us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319861.189600:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 476 00000100:00100000:3.0:1767319861.189616:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922240 00000100:00100000:3.0:1767319861.189619:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922240 00000100:00100000:3.0:1767319861.189636:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922496 00000100:00100000:3.0:1767319861.189638:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922496 00000100:00100000:3.0:1767319861.189645:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923008 00000100:00100000:3.0:1767319861.189648:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923008 00000100:00100000:3.0:1767319861.189656:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1803 00000100:00100000:3.0:1767319861.189661:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982922240:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319861.189670:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319861.189693:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982922240:12345-0@lo:38:kworker.0 Request processed in 31us (327us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319861.189699:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1803 00000100:00100000:3.0:1767319861.189709:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804 00000100:00100000:3.0:1767319861.189713:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982922496:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319861.189721:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319861.189740:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982922496:12345-0@lo:38:kworker.0 Request processed in 26us (332us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319861.189745:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804 00000100:00100000:3.0:1767319861.189756:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1805 00000100:00100000:3.0:1767319861.189759:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982923008:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319861.189767:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319861.189787:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982923008:12345-0@lo:38:kworker.0 Request processed in 26us (304us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319861.189796:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1805 00000100:00080000:3.0:1767319861.189840:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319861.189845:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319861.189848:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982922240:0@lo:38:kworker.0 00000100:00080000:3.0:1767319861.189857:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319861.189860:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319861.189863:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982922496:0@lo:38:kworker.0 00000100:00080000:3.0:1767319861.189870:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319861.189873:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319861.189876:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982922624:0@lo:38:kworker.0 00000100:00080000:3.0:1767319861.189883:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319861.189886:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319861.189889:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982923008:0@lo:38:kworker.0 00010000:00080000:3.0:1767319861.189924:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543e680 x1853163982922368/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319877 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319861.189950:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982922368:0@lo:400:kworker.0 00000100:00100000:0.0:1767319861.190119:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922752 00000100:00100000:0.0:1767319861.190124:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922752 00000100:00100000:0.0:1767319861.190170:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 851 00000100:00100000:0.0:1767319861.190177:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982922752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319861.190222:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982922752:12345-0@lo:400:kworker.0 Request processed in 46us (1229us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319861.190228:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 851 00000100:00100000:0.0:1767319861.190263:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982922752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.190698:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982922112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.190740:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319861.190784:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319861.190791:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982922880:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.190815:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319861.190845:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922112 00000100:00100000:2.0:1767319861.190849:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922112 00000100:00100000:2.0:1767319861.190884:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1368 00000100:00100000:2.0:1767319861.190892:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982922112:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.190942:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982922112:12345-0@lo:400:kworker.0 Request processed in 39us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319861.190949:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1368 00000100:00100000:2.0:1767319861.190980:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982922880 00000100:00100000:2.0:1767319861.190984:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982922880 00000100:00080000:3.0:1767319861.191711:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319861.191718:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319861.191720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:3.0:1767319861.191776:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396f480 x1853163982922112/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319877 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319861.191788:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982922112:0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.191893:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 852 00000100:00100000:2.0:1767319861.191905:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982922880:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319861.191980:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982922880:12345-0@lo:400:kworker.0 Request processed in 79us (1163us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319861.191989:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 852 00000100:00100000:3.0:1767319861.192055:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982922880:0@lo:400:kworker.0 00010000:00010000:2.0:1767319861.381455:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319862.156561:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319862.156702:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915648640 offset 0 length 520: 4 00000100:00100000:0.0:1767319862.156737:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319862.156850:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648896 00000100:00100000:1.0:1767319862.156861:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648896 00000100:00100000:1.0:1767319862.156953:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915648768 00000100:00100000:1.0:1767319862.156957:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915648768 00000100:00080000:2.0:1767319862.156962:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319862.156979:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1369 00000100:00100000:1.0:1767319862.156989:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915648896:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319862.157077:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915648896:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (533us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319862.157084:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1369 00000100:00100000:1.0:1767319862.157103:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1806 00000100:00100000:1.0:1767319862.157109:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915648768:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319862.157130:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319862.157155:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915648768:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (418us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319862.157163:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1806 00010000:00010000:2.0:1767319862.404617:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319863.428517:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319863.748843:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982923136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319863.748934:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319863.749076:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923136 00000100:00100000:3.0:1767319863.749085:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923136 00000100:00100000:3.0:1767319863.749153:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924 00000100:00100000:3.0:1767319863.749163:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982923136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319863.749191:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319863.749255:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982923136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (320us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319863.749262:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924 00000100:00100000:3.0:1767319863.749346:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982923136:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319863.749674:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319863.751000:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319863.751018:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982923264:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319863.751079:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319863.751238:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923264 00000100:00100000:0.0:1767319863.751246:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923264 00000100:00100000:0.0:1767319863.751338:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 925 00000100:00100000:0.0:1767319863.751348:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982923264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319863.751374:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319863.751418:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982923264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (338us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319863.751425:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 925 00000100:00100000:0.0:1767319863.751462:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982923264:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767319863.751818:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319864.452634:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319865.284548:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319840/real 1767319840] req@ffff9346059d5880 x1853163982917888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319865.284587:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346059d5880 x1853163982917888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319865.284607:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319865.284635:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319865.284639:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982917888:0@lo:250:kworker.0 00000100:00080000:3.0:1767319865.284715:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319865.284721:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767319865.284724:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319865.284729:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5089 00000100:00080000:3.0:1767319865.284733:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319865.284740:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319865.284788:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163982923392/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319865.284852:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319865.284863:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319865.284865:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319865.284869:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5109 00000100:00080000:3.0:1767319865.284871:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319865.284875:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319865.284890:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa300 x1853163982923648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319865.284907:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767319865.284915:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319865.284919:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319865.284920:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319865.284925:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5109 00000100:00080000:3.0:1767319865.284927:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319865.284931:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319865.284934:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982923520:0@lo:400:kworker.0 00000100:00100000:3.0:1767319865.284942:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ea00 x1853163982923904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319865.284950:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319865.284953:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319865.284954:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319865.284957:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5109 00000100:00080000:3.0:1767319865.284959:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319865.284962:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319865.284971:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d880 x1853163982924032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319865.284988:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319865.284991:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319865.284993:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319865.284996:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5109 00000100:00080000:3.0:1767319865.284998:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319865.285002:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319865.285010:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319865.285010:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ed80 x1853163982924416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319865.285034:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982924160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319865.285065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319865.285071:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982924288:0@lo:400:kworker.0 00000100:00100000:3.0:1767319865.285086:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319865.285091:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982923776:0@lo:400:kworker.0 00000100:00100000:2.0:1767319865.285092:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319865.285111:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319865.285175:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924160 00000100:00100000:0.0:1767319865.285186:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924160 00000100:00100000:0.0:1767319865.285259:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924288 00000100:00100000:0.0:1767319865.285263:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924288 00000100:00100000:0.0:1767319865.285309:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 853 00000100:00100000:0.0:1767319865.285318:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982924160:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319865.285457:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319865.285462:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319865.285470:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982923392:0@lo:250:kworker.0 00000100:00100000:0.0:1767319865.285524:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982924160:12345-0@lo:400:kworker.0 Request processed in 207us (457us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319865.285531:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 853 00000100:00100000:0.0:1767319865.285548:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 854 00000100:00100000:0.0:1767319865.285554:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982924288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.285585:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982924288:12345-0@lo:400:kworker.0 Request processed in 32us (492us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319865.285591:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 854 00000100:00100000:0.0:1767319865.285625:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982924160:0@lo:400:kworker.0 00000100:00100000:0.0:1767319865.285643:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982924288:0@lo:400:kworker.0 00000400:00000100:1.0:1767319865.292017:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982923392 offset 0 length 520: 4 00000100:00100000:1.0:1767319865.292074:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982923648:0@lo:38:kworker.0 00000100:00100000:1.0:1767319865.292125:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319865.300319:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982923904:0@lo:38:kworker.0 00000100:00100000:1.0:1767319865.300398:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319865.301456:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923520 00000100:00100000:2.0:1767319865.301470:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923520 00000100:00100000:2.0:1767319865.301542:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923776 00000100:00100000:2.0:1767319865.301546:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923776 00000100:00100000:2.0:1767319865.301659:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923648 00000100:00100000:2.0:1767319865.301663:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923648 00000100:00100000:2.0:1767319865.301679:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1370 00000100:00100000:2.0:1767319865.301689:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982923520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319865.302457:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982923520:12345-0@lo:400:kworker.0 Request processed in 767us (17443us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319865.302466:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1370 00000100:00100000:2.0:1767319865.302487:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1371 00000100:00100000:2.0:1767319865.302496:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982923776:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319865.302537:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982923776:12345-0@lo:400:kworker.0 Request processed in 43us (17425us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319865.302543:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1371 00000100:00100000:2.0:1767319865.302878:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1807 00000100:00100000:2.0:1767319865.302883:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982923648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319865.302903:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319865.303019:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982923648:12345-0@lo:38:kworker.0 Request processed in 133us (10891us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319865.303026:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1807 00010000:00080000:2.0:1767319865.303090:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb480 x1853163982923776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319881 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319865.303106:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982923776:0@lo:400:kworker.0 00010000:00080000:2.0:1767319865.303159:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9880 x1853163982923520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319881 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319865.303170:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982923520:0@lo:400:kworker.0 00000100:00080000:2.0:1767319865.304632:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319865.305389:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982924032:0@lo:38:kworker.0 00000100:00100000:1.0:1767319865.305473:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319865.305498:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982924416:0@lo:38:kworker.0 00000100:00100000:1.0:1767319865.305523:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319865.305566:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319865.305572:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319865.305576:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982923648:0@lo:38:kworker.0 00000100:00100000:2.0:1767319865.305790:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924032 00000100:00100000:2.0:1767319865.305798:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924032 00000100:00100000:2.0:1767319865.305846:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 477 00000100:00100000:2.0:1767319865.305853:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982924032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319865.305882:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319865.305941:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982924032:12345-0@lo:38:kworker.0 Request processed in 87us (468us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319865.305952:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 477 00000100:00080000:1.0:1767319865.306258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319865.306263:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319865.306267:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982924032:0@lo:38:kworker.0 00000100:00100000:3.0:1767319865.306375:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982923904 00000100:00100000:3.0:1767319865.306380:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982923904 00000100:00100000:3.0:1767319865.306400:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924416 00000100:00100000:3.0:1767319865.306402:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924416 00000100:00100000:3.0:1767319865.306426:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1808 00000100:00100000:3.0:1767319865.306431:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982923904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319865.306444:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319865.306481:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982923904:12345-0@lo:38:kworker.0 Request processed in 49us (6082us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319865.306487:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1808 00000100:00100000:3.0:1767319865.306499:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1809 00000100:00100000:3.0:1767319865.306503:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982924416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319865.306512:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319865.306532:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982924416:12345-0@lo:38:kworker.0 Request processed in 27us (1009us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319865.306537:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1809 00000100:00080000:1.0:1767319865.307211:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319865.307217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319865.307221:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982923904:0@lo:38:kworker.0 00000100:00080000:1.0:1767319865.307229:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319865.307233:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319865.307235:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982924416:0@lo:38:kworker.0 00010000:00010000:2.0:1767319865.477989:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319866.500568:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319867.275382:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319867.275547:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649152 00000100:00100000:0.0:1767319867.275558:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649152 00000100:00100000:0.0:1767319867.275646:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1372 00000100:00100000:0.0:1767319867.275656:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915649152:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319867.275746:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915649152:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (375us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319867.275765:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1372 00000100:00100000:0.0:1767319867.276448:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319867.276513:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649024 00000100:00100000:1.0:1767319867.276520:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649024 00000100:00100000:1.0:1767319867.276559:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1810 00000100:00100000:1.0:1767319867.276565:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915649024:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319867.276590:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319867.276649:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915649024:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (203us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319867.276656:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1810 00000100:00080000:2.0:1767319867.276991:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319867.525198:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319868.549906:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319868.868840:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982924544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319868.868935:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319868.869046:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924544 00000100:00100000:3.0:1767319868.869058:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924544 00000100:00100000:3.0:1767319868.869144:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926 00000100:00100000:3.0:1767319868.869154:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982924544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319868.869200:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319868.869249:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982924544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319868.869257:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926 00000100:00100000:2.0:1767319868.869755:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319868.869770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982924672:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319868.869826:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319868.870061:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924672 00000100:00100000:1.0:1767319868.870068:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924672 00000100:00100000:1.0:1767319868.870118:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927 00000100:00100000:1.0:1767319868.870127:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982924672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319868.870148:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319868.870200:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982924672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (373us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319868.870206:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927 00000100:00100000:2.0:1767319868.870864:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982924672:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319868.870952:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319868.870958:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319868.870988:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982924544:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319869.572598:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319870.404414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.404425:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319870.404517:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767319870.404536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.404540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.404544:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.404552:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.404557:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.404563:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319870.404588:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55500 x1853163982924928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319870.404637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.404641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.404643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.404646:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.404649:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.404653:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319870.404662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163982925184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319870.404670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.404673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.404686:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.404689:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5113 00000100:00080000:1.0:1767319870.404692:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.404696:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319870.404708:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55c00 x1853163982925312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319870.404789:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982924928:0@lo:38:kworker.0 00000100:00100000:0.0:1767319870.404867:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.404908:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982925184:0@lo:38:kworker.0 00000100:00100000:0.0:1767319870.404929:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.404946:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982925312:0@lo:38:kworker.0 00000100:00100000:0.0:1767319870.404972:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.405005:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982925440:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405032:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.405055:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319870.405060:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982925056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405104:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.405139:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925440 00000100:00100000:0.0:1767319870.405149:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925440 00000100:00100000:0.0:1767319870.405217:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 855 00000100:00100000:0.0:1767319870.405227:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982925440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405315:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982925440:12345-0@lo:400:kworker.0 Request processed in 89us (281us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319870.405323:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 855 00000100:00080000:0.0:1767319870.405353:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319870.405381:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924928 00000100:00100000:0.0:1767319870.405385:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924928 00000100:00100000:0.0:1767319870.405405:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925184 00000100:00100000:0.0:1767319870.405408:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925184 00000100:00100000:0.0:1767319870.405422:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925056 00000100:00100000:0.0:1767319870.405425:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925056 00000100:00100000:0.0:1767319870.405446:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1373 00000100:00100000:0.0:1767319870.405453:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982925056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405484:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982925056:12345-0@lo:400:kworker.0 Request processed in 32us (379us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319870.405490:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1373 00000100:00100000:0.0:1767319870.405502:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1811 00000100:00100000:0.0:1767319870.405506:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982924928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319870.405524:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319870.405557:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982924928:12345-0@lo:38:kworker.0 Request processed in 49us (690us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319870.405563:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1811 00000100:00100000:0.0:1767319870.405575:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812 00000100:00100000:0.0:1767319870.405578:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982925184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319870.405588:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319870.405611:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982925184:12345-0@lo:38:kworker.0 Request processed in 31us (680us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319870.405616:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812 00000100:00080000:0.0:1767319870.405633:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319870.405655:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319870.405659:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319870.405662:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982924928:0@lo:38:kworker.0 00000100:00080000:0.0:1767319870.405671:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319870.405675:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319870.405677:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982925184:0@lo:38:kworker.0 00010000:00080000:0.0:1767319870.405707:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57480 x1853163982925056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319886 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319870.405718:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982925056:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405727:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982925440:0@lo:400:kworker.0 00000100:00100000:0.0:1767319870.405785:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925312 00000100:00100000:0.0:1767319870.405788:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925312 00000100:00100000:0.0:1767319870.405806:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 478 00000100:00100000:0.0:1767319870.405810:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982925312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319870.405821:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319870.405850:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982925312:12345-0@lo:38:kworker.0 Request processed in 38us (878us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319870.405854:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319870.405858:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 478 00000100:00100000:2.0:1767319870.405871:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982925568:0@lo:400:kworker.0 00000100:00080000:0.0:1767319870.405884:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319870.405889:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319870.405892:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982925312:0@lo:38:kworker.0 00000100:00100000:2.0:1767319870.405927:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319870.405967:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925568 00000100:00100000:2.0:1767319870.405971:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925568 00000100:00100000:2.0:1767319870.406011:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 856 00000100:00100000:2.0:1767319870.406024:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982925568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319870.406061:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982925568:12345-0@lo:400:kworker.0 Request processed in 43us (135us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319870.406083:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 856 00000100:00100000:2.0:1767319870.406111:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982925568:0@lo:400:kworker.0 00000100:00080000:0.0:1767319870.406206:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319870.407218:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982924800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.407269:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319870.407346:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982924800 00000100:00100000:0.0:1767319870.407353:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982924800 00000100:00080000:1.0:1767319870.407469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319870.407475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319870.407478:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319870.407485:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5113 00000100:00100000:0.0:1767319870.407486:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1374 00000100:00080000:1.0:1767319870.407490:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319870.407498:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319870.407500:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982924800:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319870.407531:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982925696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319870.407554:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982924800:12345-0@lo:400:kworker.0 Request processed in 57us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319870.407562:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1374 00000100:00100000:1.0:1767319870.407586:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982925696:0@lo:38:kworker.0 00000100:00100000:1.0:1767319870.407622:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319870.407643:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925696 00000100:00100000:3.0:1767319870.407649:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925696 00000100:00100000:3.0:1767319870.407692:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1813 00000100:00100000:3.0:1767319870.407750:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982925696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319870.407778:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319870.407823:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982925696:12345-0@lo:38:kworker.0 Request processed in 124us (201us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767319870.407876:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319870.407881:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319870.407884:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982925696:0@lo:38:kworker.0 00000100:00080000:2.0:1767319870.408040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319870.408073:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54e00 x1853163982924800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319886 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319870.408085:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982924800:0@lo:400:kworker.0 00000100:00100000:3.0:1767319870.408350:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1813 00010000:00010000:2.0:1767319870.596725:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319871.620674:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319872.395625:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319872.395700:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649408 00000100:00100000:0.0:1767319872.395712:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649408 00000100:00100000:0.0:1767319872.395828:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1375 00000100:00100000:0.0:1767319872.395848:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915649408:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319872.395934:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915649408:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (321us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319872.395943:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1375 00000100:00080000:3.0:1767319872.396407:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319872.396489:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319872.397805:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649280 00000100:00100000:1.0:1767319872.397820:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649280 00000100:00100000:1.0:1767319872.397920:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1814 00000100:00100000:1.0:1767319872.397928:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915649280:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319872.397952:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319872.398024:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915649280:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (1535us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319872.398031:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1814 00010000:00010000:1.0:1767319872.644798:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319873.668711:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319873.988803:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319873.988918:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319873.988941:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982925952:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319873.989018:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319873.989100:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925952 00000100:00100000:3.0:1767319873.989108:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925952 00000100:00100000:3.0:1767319873.989172:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 928 00000100:00100000:3.0:1767319873.989183:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982925952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319873.989219:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319873.989267:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982925952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319873.989304:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 928 00000100:00080000:3.0:1767319873.989330:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319873.989373:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982925952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319873.991121:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982925824:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319873.991249:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319873.991391:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982925824 00000100:00100000:3.0:1767319873.991399:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982925824 00000100:00100000:3.0:1767319873.991519:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929 00000100:00100000:3.0:1767319873.991529:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982925824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319873.991560:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319873.991672:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982925824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 144us (423us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319873.991680:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929 00000100:00100000:2.0:1767319873.996442:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982925824:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319873.996765:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319874.693875:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319875.524805:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319875.524817:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319875.524893:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319875.524897:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319875.524901:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319875.524910:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5118 00000100:00080000:3.0:1767319875.524915:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319875.524922:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319875.524943:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982926080:0@lo:400:kworker.0 00000100:00100000:3.0:1767319875.524949:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982926208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319875.524997:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982926208:0@lo:38:kworker.0 00000100:00080000:3.0:1767319875.524998:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319875.525005:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319875.525008:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319875.525012:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5118 00000100:00080000:3.0:1767319875.525015:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319875.525019:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319875.525026:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319875.525036:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014e00 x1853163982926464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319875.525047:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319875.525050:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319875.525053:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319875.525057:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5118 00000100:00080000:3.0:1767319875.525060:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767319875.525061:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926080 00000100:00080000:3.0:1767319875.525065:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319875.525070:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926080 00000100:00100000:3.0:1767319875.525081:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016a00 x1853163982926592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319875.525108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319875.525115:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319875.525120:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319875.525134:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319875.525138:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5118 00000100:00080000:3.0:1767319875.525141:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319875.525145:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319875.525153:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982926848:0@lo:400:kworker.0 00000100:00100000:3.0:1767319875.525161:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016300 x1853163982926976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319875.525186:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982926336:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.525191:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.525228:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926848 00000100:00100000:2.0:1767319875.525232:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926848 00000100:00100000:3.0:1767319875.525232:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982926720:0@lo:400:kworker.0 00000100:00100000:3.0:1767319875.525264:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.525296:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 857 00000100:00100000:2.0:1767319875.525306:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982926848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319875.525324:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926720 00000100:00100000:3.0:1767319875.525329:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926720 00000100:00100000:2.0:1767319875.525356:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982926848:12345-0@lo:400:kworker.0 Request processed in 52us (166us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319875.525364:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 858 00000100:00100000:3.0:1767319875.525372:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982926720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319875.525414:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982926720:12345-0@lo:400:kworker.0 Request processed in 43us (151us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319875.527981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982926464:0@lo:38:kworker.0 00000100:00100000:1.0:1767319875.528051:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.532315:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 857 00000100:00100000:2.0:1767319875.532410:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982926848:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.532554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982926720:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.532611:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.532681:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926464 00000100:00100000:2.0:1767319875.532690:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926464 00000100:00100000:2.0:1767319875.532714:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926336 00000100:00100000:2.0:1767319875.532716:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926336 00000100:00100000:2.0:1767319875.532741:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1376 00000100:00100000:2.0:1767319875.532749:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982926080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319875.532815:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982926592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319875.532831:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982926080:12345-0@lo:400:kworker.0 Request processed in 81us (7808us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319875.532838:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1376 00000100:00100000:2.0:1767319875.532851:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1377 00000100:00100000:2.0:1767319875.532858:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982926336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319875.532866:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319875.533050:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982926336:12345-0@lo:400:kworker.0 Request processed in 192us (439us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319875.533057:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1377 00000100:00100000:2.0:1767319875.533069:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1815 00000100:00100000:2.0:1767319875.533073:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982926464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319875.533091:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319875.533118:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982926464:12345-0@lo:38:kworker.0 Request processed in 44us (5068us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319875.533124:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1815 00000100:00080000:2.0:1767319875.533142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319875.533147:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319875.533150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767319875.533169:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016680 x1853163982926336/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319891 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319875.533181:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982926336:0@lo:400:kworker.0 00000100:00100000:2.0:1767319875.533208:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926592 00000100:00100000:2.0:1767319875.533212:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926592 00000100:00100000:2.0:1767319875.533230:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 479 00000100:00100000:2.0:1767319875.533234:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982926592:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319875.533244:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319875.533267:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982926592:12345-0@lo:38:kworker.0 Request processed in 31us (403us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319875.533295:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 479 00010000:00080000:2.0:1767319875.533338:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017100 x1853163982926080/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319891 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319875.533354:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982926080:0@lo:400:kworker.0 00000100:00080000:1.0:1767319875.537340:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319875.537356:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319875.537362:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982926592:0@lo:38:kworker.0 00000100:00100000:1.0:1767319875.537377:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982926976:0@lo:38:kworker.0 00000100:00100000:1.0:1767319875.537442:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319875.537496:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319875.537500:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319875.537503:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982926464:0@lo:38:kworker.0 00000100:00100000:2.0:1767319875.537847:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926976 00000100:00100000:2.0:1767319875.537856:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926976 00000100:00100000:2.0:1767319875.537916:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816 00000100:00100000:2.0:1767319875.537925:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982926976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319875.537951:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319875.538005:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982926976:12345-0@lo:38:kworker.0 Request processed in 79us (561us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319875.538013:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816 00000100:00080000:1.0:1767319875.538097:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319875.538101:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319875.538105:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982926976:0@lo:38:kworker.0 00000100:00100000:0.0:1767319875.540243:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982926208 00000100:00100000:0.0:1767319875.540262:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982926208 00000100:00100000:0.0:1767319875.540355:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1817 00000100:00100000:0.0:1767319875.540365:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982926208:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319875.540386:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 858 00010000:00020000:0.0:1767319875.540392:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319875.540480:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982926208:12345-0@lo:38:kworker.0 Request processed in 114us (15372us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319875.540488:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1817 00000100:00080000:1.0:1767319875.540513:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319875.540520:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319875.540525:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982926208:0@lo:38:kworker.0 00010000:00010000:1.0:1767319875.716627:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319876.740702:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319877.515911:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319877.515950:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319877.516031:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649664 00000100:00100000:2.0:1767319877.516042:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649664 00000100:00100000:2.0:1767319877.516139:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649536 00000100:00100000:2.0:1767319877.516143:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649536 00000100:00100000:1.0:1767319877.516158:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1378 00000100:00100000:1.0:1767319877.516169:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915649664:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319877.516173:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1818 00000100:00100000:2.0:1767319877.516184:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915649536:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319877.516208:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319877.516240:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915649664:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 73us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319877.516250:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1378 00000100:00100000:2.0:1767319877.516251:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915649536:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (296us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319877.516262:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1818 00000100:00080000:0.0:1767319877.517195:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319877.764569:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319878.788467:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319879.108555:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319879.108581:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982927104:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319879.108665:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319879.110340:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982927232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319879.110443:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319879.124295:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927104 00000100:00100000:1.0:1767319879.124327:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927104 00000100:00100000:1.0:1767319879.124589:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927232 00000100:00100000:1.0:1767319879.124594:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927232 00000100:00100000:2.0:1767319879.124681:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 930 00000100:00100000:2.0:1767319879.124718:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982927104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319879.124764:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319879.124896:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982927104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 204us (16230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319879.124920:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 930 00000100:00100000:2.0:1767319879.125043:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982927104:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767319879.126495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319879.126508:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319879.126766:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 931 00000100:00100000:1.0:1767319879.126781:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982927232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319879.126817:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319879.126898:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982927232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 122us (16459us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319879.126905:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 931 00000100:00100000:2.0:1767319879.127384:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982927232:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319879.818417:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319880.645315:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.645325:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319880.645391:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.645396:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.645400:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.645409:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.645414:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.645421:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319880.645444:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163982927488/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319880.645472:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.645475:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.645477:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.645480:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.645483:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.645487:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319880.645496:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163982927744/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319880.645506:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.645509:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.645510:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.645513:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.645515:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.645518:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319880.645530:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982927872/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319880.645550:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319880.645553:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319880.645555:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319880.645558:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5124 00000100:00080000:3.0:1767319880.645561:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319880.645564:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319880.645790:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982928128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319880.645857:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319880.645920:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319880.645940:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982927360:0@lo:400:kworker.0 00000100:00100000:0.0:1767319880.646039:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319880.646092:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928128 00000100:00100000:0.0:1767319880.646101:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928128 00000100:00100000:0.0:1767319880.646145:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 859 00000100:00100000:0.0:1767319880.646153:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982928128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319880.646205:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982928128:12345-0@lo:400:kworker.0 Request processed in 52us (346us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319880.646212:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 859 00000100:00100000:0.0:1767319880.646253:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982928128:0@lo:400:kworker.0 00000100:00100000:0.0:1767319880.646308:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927360 00000100:00100000:0.0:1767319880.646312:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927360 00000100:00100000:0.0:1767319880.646347:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1379 00000100:00100000:0.0:1767319880.646353:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982927360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319880.646390:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982927360:12345-0@lo:400:kworker.0 Request processed in 37us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319880.646396:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1379 00000100:00080000:0.0:1767319880.646412:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319880.646428:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015180 x1853163982927360/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319896 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319880.646529:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982928256/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319880.646907:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982927488:0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.647029:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319880.647066:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982927744:0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.647090:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319880.647105:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982927872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.647126:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319880.647147:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982928256:0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.647165:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319880.647195:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927488 00000100:00100000:1.0:1767319880.647201:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927488 00000100:00100000:1.0:1767319880.647218:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927744 00000100:00100000:1.0:1767319880.647220:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927744 00000100:00100000:1.0:1767319880.647227:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928256 00000100:00100000:1.0:1767319880.647229:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928256 00000100:00100000:1.0:1767319880.647239:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1819 00000100:00100000:1.0:1767319880.647244:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982927488:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319880.647262:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319880.647320:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982927488:12345-0@lo:38:kworker.0 Request processed in 74us (291us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319880.647326:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1819 00000100:00100000:1.0:1767319880.647341:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1820 00000100:00100000:1.0:1767319880.647345:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982927744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319880.647354:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319880.647377:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982927744:12345-0@lo:38:kworker.0 Request processed in 30us (286us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319880.647383:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1820 00000100:00100000:1.0:1767319880.647394:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1821 00000100:00100000:1.0:1767319880.647398:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982928256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319880.647406:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319880.647747:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982927360:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.648007:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982927616:0@lo:400:kworker.0 00000100:00100000:3.0:1767319880.648669:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982928000:0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.648759:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.648803:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927616 00000100:00100000:2.0:1767319880.648811:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927616 00000100:00100000:2.0:1767319880.648858:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1380 00000100:00100000:2.0:1767319880.648867:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982927616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.648913:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982927616:12345-0@lo:400:kworker.0 Request processed in 48us (154us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319880.648920:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1380 00000100:00100000:2.0:1767319880.648963:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982927872 00000100:00100000:2.0:1767319880.648968:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982927872 00000100:00100000:2.0:1767319880.648987:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 480 00000100:00100000:2.0:1767319880.648993:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982927872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319880.649009:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319880.649037:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982927872:12345-0@lo:38:kworker.0 Request processed in 43us (1909us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319880.649043:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 480 00010000:00080000:2.0:1767319880.649084:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015f80 x1853163982927616/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319896 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319880.649098:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982927616:0@lo:400:kworker.0 00000100:00080000:2.0:1767319880.649138:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319880.652674:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.652687:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.652691:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982927488:0@lo:38:kworker.0 00000100:00080000:0.0:1767319880.652709:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.652713:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.652717:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982927744:0@lo:38:kworker.0 00000100:00080000:0.0:1767319880.652730:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.652735:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.652738:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982927872:0@lo:38:kworker.0 00000100:00100000:1.0:1767319880.653135:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982928256:12345-0@lo:38:kworker.0 Request processed in 5732us (5964us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319880.653148:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1821 00000100:00080000:0.0:1767319880.655304:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319880.655318:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319880.655323:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982928256:0@lo:38:kworker.0 00000100:00100000:3.0:1767319880.655475:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319880.655914:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928000 00000100:00100000:2.0:1767319880.656211:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928000 00000100:00100000:2.0:1767319880.656346:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 860 00000100:00100000:2.0:1767319880.656356:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982928000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319880.656431:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982928000:12345-0@lo:400:kworker.0 Request processed in 77us (956us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319880.656439:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 860 00000100:00100000:3.0:1767319880.656507:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982928000:0@lo:400:kworker.0 00010000:00010000:2.0:1767319880.836510:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319881.862362:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319882.640263:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319882.641403:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319882.644400:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649920 00000100:00100000:2.0:1767319882.644418:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649920 00000100:00100000:2.0:1767319882.644493:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915649792 00000100:00100000:2.0:1767319882.644497:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915649792 00000100:00100000:2.0:1767319882.644524:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1381 00000100:00100000:2.0:1767319882.644544:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915649920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319882.644566:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1822 00000100:00100000:3.0:1767319882.644576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915649792:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319882.644600:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319882.644644:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915649920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 108us (4391us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319882.644644:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915649792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 65us (3243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319882.644661:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1822 00000100:00080000:0.0:1767319882.645110:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319882.647375:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1381 00010000:00010000:2.0:1767319882.884430:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319883.908557:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319884.228634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982928384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319884.228767:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319884.228932:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928384 00000100:00100000:2.0:1767319884.228943:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928384 00000100:00100000:2.0:1767319884.229015:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 932 00000100:00100000:2.0:1767319884.229027:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982928384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319884.229068:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319884.229125:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982928384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (358us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319884.229134:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 932 00000100:00080000:0.0:1767319884.229203:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319884.229248:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982928384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319884.229996:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319884.230011:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982928512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319884.230060:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319884.230196:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928512 00000100:00100000:3.0:1767319884.230202:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928512 00000100:00100000:3.0:1767319884.230260:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933 00000100:00080000:0.0:1767319884.230261:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319884.230292:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982928512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319884.230319:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319884.230373:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982928512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (311us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319884.230381:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933 00000100:00100000:0.0:1767319884.230458:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982928512:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767319884.935167:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319885.764465:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319885.764488:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319885.764570:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319885.764575:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319885.764579:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319885.764589:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5129 00000100:00080000:1.0:1767319885.764594:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319885.764602:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319885.764634:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960a80 x1853163982928768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319885.764687:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319885.764690:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319885.764692:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319885.764697:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5129 00000100:00080000:1.0:1767319885.764699:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319885.764703:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319885.764712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962300 x1853163982929024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319885.764720:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319885.764723:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319885.764725:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319885.764728:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5129 00000100:00080000:1.0:1767319885.764730:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319885.764734:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319885.764744:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163982929152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319885.764763:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319885.764766:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319885.764768:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319885.764771:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5129 00000100:00080000:1.0:1767319885.764774:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319885.764777:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319885.764787:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f962680 x1853163982929536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319885.764821:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982929280:0@lo:400:kworker.0 00000100:00100000:1.0:1767319885.764889:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319885.764971:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319885.764978:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982928896:0@lo:400:kworker.0 00000100:00100000:1.0:1767319885.765005:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319885.765104:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982928768:0@lo:38:kworker.0 00000100:00100000:3.0:1767319885.765134:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319885.765165:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982929024:0@lo:38:kworker.0 00000100:00100000:3.0:1767319885.765184:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319885.765202:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982929152:0@lo:38:kworker.0 00000100:00100000:3.0:1767319885.765223:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319885.765242:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982929536:0@lo:38:kworker.0 00000100:00100000:3.0:1767319885.765259:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319885.765308:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928896 00000100:00100000:3.0:1767319885.765316:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928896 00000100:00100000:3.0:1767319885.765368:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928768 00000100:00100000:3.0:1767319885.765371:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928768 00000100:00100000:3.0:1767319885.765381:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929024 00000100:00100000:3.0:1767319885.765383:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929024 00000100:00100000:3.0:1767319885.765389:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929536 00000100:00100000:3.0:1767319885.765391:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929536 00000100:00100000:3.0:1767319885.765402:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1382 00000100:00100000:3.0:1767319885.765411:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982928896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319885.765462:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982928896:12345-0@lo:400:kworker.0 Request processed in 52us (455us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319885.765468:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1382 00000100:00100000:3.0:1767319885.765502:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1823 00000100:00100000:3.0:1767319885.765506:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982928768:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319885.765524:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319885.765554:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982928768:12345-0@lo:38:kworker.0 Request processed in 47us (419us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319885.765560:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1823 00000100:00100000:3.0:1767319885.765571:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824 00000100:00100000:3.0:1767319885.765574:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982929024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319885.765583:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319885.765603:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982929024:12345-0@lo:38:kworker.0 Request processed in 27us (418us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319885.765609:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824 00000100:00100000:3.0:1767319885.765618:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1825 00000100:00100000:3.0:1767319885.765622:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982929536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319885.765630:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319885.765648:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982929536:12345-0@lo:38:kworker.0 Request processed in 26us (389us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319885.765653:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1825 00000100:00100000:3.0:1767319885.765672:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929152 00000100:00100000:3.0:1767319885.765676:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929152 00000100:00100000:3.0:1767319885.765694:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 481 00000100:00100000:3.0:1767319885.765698:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982929152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319885.765708:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319885.765731:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982929152:12345-0@lo:38:kworker.0 Request processed in 31us (507us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319885.765738:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 481 00000100:00080000:3.0:1767319885.765754:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319885.765782:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319885.765787:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319885.765791:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982928768:0@lo:38:kworker.0 00000100:00080000:3.0:1767319885.765800:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319885.765804:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319885.765807:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982929024:0@lo:38:kworker.0 00000100:00080000:3.0:1767319885.765814:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319885.765817:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319885.765819:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982929152:0@lo:38:kworker.0 00000100:00080000:3.0:1767319885.765826:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319885.765829:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319885.765831:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982929536:0@lo:38:kworker.0 00010000:00080000:3.0:1767319885.765869:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963480 x1853163982928896/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319901 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319885.765879:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982928896:0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.766467:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982929408:0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.766545:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319885.766590:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319885.766600:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982928640:0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.766626:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319885.766660:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929280 00000100:00100000:2.0:1767319885.766665:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929280 00000100:00100000:2.0:1767319885.766698:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929408 00000100:00100000:2.0:1767319885.766702:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929408 00000100:00100000:2.0:1767319885.766721:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 861 00000100:00100000:2.0:1767319885.766728:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982929280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.766774:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982929280:12345-0@lo:400:kworker.0 Request processed in 47us (1886us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319885.766781:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 861 00000100:00100000:2.0:1767319885.766796:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 862 00000100:00100000:2.0:1767319885.766803:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982929408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.766835:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982929408:12345-0@lo:400:kworker.0 Request processed in 34us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319885.766842:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 862 00000100:00100000:2.0:1767319885.766858:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982928640 00000100:00100000:2.0:1767319885.766862:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982928640 00000100:00100000:2.0:1767319885.766889:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1383 00000100:00100000:2.0:1767319885.766895:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982928640:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319885.767040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319885.767112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982929280:0@lo:400:kworker.0 00010000:00080000:3.0:1767319885.767144:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55500 x1853163982928640/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319901 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319885.767152:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982928640:0@lo:400:kworker.0 00000100:00100000:3.0:1767319885.767160:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982929408:0@lo:400:kworker.0 00000100:00100000:2.0:1767319885.767627:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982928640:12345-0@lo:400:kworker.0 Request processed in 729us (997us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319885.767636:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1383 00010000:00010000:2.0:1767319885.956521:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319886.758446:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319886.758633:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650304 00000100:00100000:2.0:1767319886.758645:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650304 00000100:00100000:2.0:1767319886.758721:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1384 00000100:00100000:2.0:1767319886.758730:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915650304:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319886.758817:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915650304:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (379us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319886.758825:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1384 00000100:00080000:3.0:1767319886.759159:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767319886.762671:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915650048 offset 0 length 520: 4 00000100:00100000:0.0:1767319886.762796:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319886.762899:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650176 00000100:00100000:2.0:1767319886.762907:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650176 00000100:00100000:2.0:1767319886.762948:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1826 00000100:00100000:2.0:1767319886.762955:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915650176:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319886.762975:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319886.763027:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915650176:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 73us (232us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319886.763032:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1826 00010000:00010000:2.0:1767319886.988042:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319888.005698:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319889.030035:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319889.357543:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982929664:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319889.357631:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319889.357817:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929664 00000100:00100000:2.0:1767319889.357828:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929664 00000100:00100000:2.0:1767319889.357900:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 934 00000100:00100000:2.0:1767319889.357911:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982929664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319889.357950:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319889.358003:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982929664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (372us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319889.358015:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 934 00000100:00080000:0.0:1767319889.358174:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319889.358217:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982929664:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319889.358988:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319889.359002:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982929792:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319889.359048:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319889.360085:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929792 00000100:00100000:3.0:1767319889.360107:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929792 00000100:00100000:3.0:1767319889.360174:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935 00000100:00100000:3.0:1767319889.360182:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982929792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319889.360213:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319889.360300:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982929792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 118us (1249us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319889.360308:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935 00000100:00100000:3.0:1767319889.360366:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982929792:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319889.360720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767319890.052865:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319890.884714:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.884726:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319890.884957:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.884962:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.884966:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.884976:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.884982:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.884990:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319890.885014:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059ce300 x1853163982930048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319890.885087:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319865/real 1767319865] req@ffff9346076fb100 x1853163982923392/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319890 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767319890.885157:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.885170:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.885172:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.885176:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.885179:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.885183:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319890.885196:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cd500 x1853163982930304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319890.885206:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.885210:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.885212:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.885215:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.885218:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.885221:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319890.885231:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cdf80 x1853163982930432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319890.885253:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319890.885256:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319890.885258:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319890.885262:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5134 00000100:00080000:2.0:1767319890.885264:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319890.885268:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319890.885309:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c700 x1853163982930816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319890.885523:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982930688:0@lo:400:kworker.0 00000100:00100000:0.0:1767319890.885594:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.885661:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319890.885668:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982929920:0@lo:400:kworker.0 00000100:00100000:0.0:1767319890.885733:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.885772:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982930560:0@lo:400:kworker.0 00000100:00100000:0.0:1767319890.885796:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.885817:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319890.885823:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982930176:0@lo:400:kworker.0 00000100:00100000:0.0:1767319890.886593:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319890.886640:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982929920 00000100:00100000:0.0:1767319890.886647:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982929920 00000100:00100000:3.0:1767319890.886700:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930688 00000100:00100000:3.0:1767319890.886705:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930688 00000100:00100000:0.0:1767319890.886706:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930176 00000100:00100000:0.0:1767319890.886708:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930176 00000100:00100000:3.0:1767319890.886729:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930560 00000100:00100000:0.0:1767319890.886731:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1385 00000100:00100000:3.0:1767319890.886731:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930560 00000100:00100000:0.0:1767319890.886738:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982929920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.886746:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 863 00000100:00100000:3.0:1767319890.886751:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982930688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319890.886830:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 864 00000100:00100000:2.0:1767319890.886849:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982930560:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319890.893792:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076fb100 x1853163982923392/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319890 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319890.893843:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982930048:0@lo:38:kworker.0 00000100:00100000:3.0:1767319890.894449:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982930688:12345-0@lo:400:kworker.0 Request processed in 7683us (8841us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319890.894464:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 863 00000100:00100000:3.0:1767319890.894527:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1386 00000100:00100000:3.0:1767319890.894542:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982930176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.894628:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982930176:12345-0@lo:400:kworker.0 Request processed in 93us (8782us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319890.894636:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1386 00000100:00100000:3.0:1767319890.894713:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982930688:0@lo:400:kworker.0 00010000:00080000:3.0:1767319890.894905:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cf480 x1853163982930176/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319906 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319890.894918:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982930176:0@lo:400:kworker.0 00000100:00080000:3.0:1767319890.894941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319890.904846:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982929920:12345-0@lo:400:kworker.0 Request processed in 18091us (19096us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319890.904865:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1385 00010000:00080000:3.0:1767319890.905105:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059cfb80 x1853163982929920/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319906 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319890.905124:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982929920:0@lo:400:kworker.0 00000100:00100000:3.0:1767319890.905474:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982930560:12345-0@lo:400:kworker.0 Request processed in 18625us (19672us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319890.905483:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 864 00000100:00100000:0.0:1767319890.905732:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982930560:0@lo:400:kworker.0 00000100:00100000:1.0:1767319890.908072:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319890.908416:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982930304:0@lo:38:kworker.0 00000100:00100000:1.0:1767319890.908474:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319890.912182:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982930432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319890.912291:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319890.912329:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982930816:0@lo:38:kworker.0 00000100:00100000:1.0:1767319890.912362:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319890.912434:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930432 00000100:00100000:2.0:1767319890.912458:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930432 00000100:00100000:2.0:1767319890.912519:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 482 00000100:00100000:2.0:1767319890.912527:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982930432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319890.912554:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319890.912618:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982930432:12345-0@lo:38:kworker.0 Request processed in 90us (327us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319890.912625:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 482 00000100:00100000:0.0:1767319890.913084:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930048 00000100:00100000:0.0:1767319890.913091:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930048 00000100:00080000:1.0:1767319890.916453:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319890.916491:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319890.916498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982923392:0@lo:250:kworker.0 00000100:00080000:1.0:1767319890.916567:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319890.916572:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319890.916575:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982930432:0@lo:38:kworker.0 00000100:00100000:1.0:1767319890.917698:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930304 00000100:00100000:1.0:1767319890.917713:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930304 00000100:00100000:1.0:1767319890.917735:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982930816 00000100:00100000:1.0:1767319890.917738:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982930816 00000100:00100000:1.0:1767319890.917755:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1827 00000100:00100000:1.0:1767319890.917764:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319890.917791:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319890.918047:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930048:12345-0@lo:38:kworker.0 Request processed in 283us (9983us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319890.918056:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1827 00000100:00100000:1.0:1767319890.918077:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828 00000100:00100000:1.0:1767319890.918082:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319890.918092:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319890.918118:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930304:12345-0@lo:38:kworker.0 Request processed in 35us (9643us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319890.918124:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828 00000100:00100000:1.0:1767319890.918139:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1829 00000100:00100000:1.0:1767319890.918144:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319890.918152:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319890.918719:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319890.918727:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319890.918732:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319890.918738:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5113 00000100:00080000:0.0:1767319890.918742:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319890.918748:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319890.918785:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163982930944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319890.918934:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982930816:12345-0@lo:38:kworker.0 Request processed in 787us (6569us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319890.918941:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1829 00000100:00100000:0.0:1767319890.918991:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982930944:0@lo:250:kworker.0 00000400:00000100:0.0:1767319890.919030:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982930944 offset 0 length 520: 4 00000100:00080000:0.0:1767319890.919083:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.919089:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.919093:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982930048:0@lo:38:kworker.0 00000100:00080000:0.0:1767319890.919105:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.919108:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.919111:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982930304:0@lo:38:kworker.0 00000100:00080000:0.0:1767319890.919119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319890.919123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319890.919125:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982930816:0@lo:38:kworker.0 00010000:00010000:1.0:1767319891.076697:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319891.858848:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319891.858889:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319891.859095:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650432 00000100:00100000:1.0:1767319891.859107:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650432 00000100:00100000:3.0:1767319891.859137:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650560 00000100:00100000:3.0:1767319891.859155:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650560 00000100:00100000:1.0:1767319891.859167:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1830 00000100:00100000:1.0:1767319891.859178:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915650432:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319891.859204:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319891.859208:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1387 00000100:00100000:3.0:1767319891.859215:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915650560:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767319891.859335:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319891.859491:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915650432:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 312us (652us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319891.859500:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1830 00000100:00100000:3.0:1767319891.859666:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915650560:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 448us (771us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319891.859674:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1387 00010000:00010000:2.0:1767319892.100993:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319893.124665:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319894.148891:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319894.468823:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319894.468859:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982931200:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319894.468886:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982931072:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319894.471985:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319894.472110:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931200 00000100:00100000:3.0:1767319894.472121:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931200 00000100:00100000:3.0:1767319894.472211:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936 00000100:00100000:3.0:1767319894.472222:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982931200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319894.472268:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319894.472359:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982931200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 137us (383us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319894.472366:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936 00000100:00080000:2.0:1767319894.472535:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319894.472587:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982931200:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319894.474953:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319894.475392:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931072 00000100:00100000:1.0:1767319894.475402:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931072 00000100:00100000:1.0:1767319894.475498:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 937 00000100:00100000:1.0:1767319894.475509:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982931072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319894.475544:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319894.475641:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982931072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 134us (697us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319894.475649:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 937 00000100:00080000:2.0:1767319894.475659:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767319894.475729:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982931072:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319895.174346:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319896.005465:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319896.005480:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319896.005561:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319896.005566:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319896.005569:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319896.005578:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5139 00000100:00080000:2.0:1767319896.005582:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319896.005589:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319896.005613:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ea00 x1853163982931456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319896.005665:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319896.005669:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319896.005671:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319896.005675:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5139 00000100:00080000:2.0:1767319896.005678:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319896.005682:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319896.005696:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3100 x1853163982931712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319896.005705:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319896.005709:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319896.005711:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319896.005715:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5139 00000100:00080000:2.0:1767319896.005717:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319896.005720:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319896.005732:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163982931840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319896.005755:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319896.005759:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319896.005762:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319896.005766:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5139 00000100:00080000:2.0:1767319896.005768:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319896.005772:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319896.005784:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163982932224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319896.005940:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982931584:0@lo:400:kworker.0 00000100:00100000:1.0:1767319896.006016:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006064:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767319896.006071:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982931968:0@lo:400:kworker.0 00000100:00100000:1.0:1767319896.006100:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006144:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982931456:0@lo:38:kworker.0 00000100:00100000:1.0:1767319896.006170:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006189:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982931712:0@lo:38:kworker.0 00000100:00100000:1.0:1767319896.006210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006228:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982931840:0@lo:38:kworker.0 00000100:00100000:1.0:1767319896.006251:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982932224:0@lo:38:kworker.0 00000100:00100000:1.0:1767319896.006330:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319896.006364:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931584 00000100:00100000:1.0:1767319896.006372:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931584 00000100:00100000:1.0:1767319896.006430:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931456 00000100:00100000:1.0:1767319896.006433:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931456 00000100:00100000:1.0:1767319896.006444:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931712 00000100:00100000:1.0:1767319896.006446:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931712 00000100:00100000:1.0:1767319896.006453:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932224 00000100:00100000:1.0:1767319896.006456:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932224 00000100:00100000:1.0:1767319896.006467:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1388 00000100:00100000:1.0:1767319896.006477:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982931584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319896.006535:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982931584:12345-0@lo:400:kworker.0 Request processed in 60us (518us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319896.006542:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1388 00000100:00100000:1.0:1767319896.006570:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1831 00000100:00100000:1.0:1767319896.006575:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982931456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319896.006597:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.006631:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982931456:12345-0@lo:38:kworker.0 Request processed in 55us (460us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.006638:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1831 00000100:00100000:1.0:1767319896.006649:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1832 00000100:00100000:1.0:1767319896.006653:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982931712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319896.006663:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.006690:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982931712:12345-0@lo:38:kworker.0 Request processed in 35us (479us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.006695:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1832 00000100:00100000:1.0:1767319896.006707:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1833 00000100:00100000:1.0:1767319896.006711:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319896.006719:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.006739:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932224:12345-0@lo:38:kworker.0 Request processed in 27us (409us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.006744:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1833 00000100:00100000:1.0:1767319896.006767:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931840 00000100:00100000:1.0:1767319896.006770:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931840 00000100:00100000:1.0:1767319896.006788:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 483 00000100:00100000:1.0:1767319896.006792:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982931840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319896.006803:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.006828:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982931840:12345-0@lo:38:kworker.0 Request processed in 34us (576us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.006834:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 483 00000100:00080000:1.0:1767319896.006852:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319896.006882:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398dc00 x1853163982931584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319912 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319896.006898:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982931584:0@lo:400:kworker.0 00000100:00080000:1.0:1767319896.006928:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319896.006933:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319896.006936:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982931456:0@lo:38:kworker.0 00000100:00080000:1.0:1767319896.006944:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319896.006947:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319896.006950:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982931712:0@lo:38:kworker.0 00000100:00080000:1.0:1767319896.006959:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319896.006962:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319896.006965:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982931840:0@lo:38:kworker.0 00000100:00080000:1.0:1767319896.006973:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319896.006976:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319896.006978:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982932224:0@lo:38:kworker.0 00000100:00100000:0.0:1767319896.008429:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982931328:0@lo:400:kworker.0 00000100:00100000:0.0:1767319896.008508:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319896.008585:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319896.008593:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982932096:0@lo:400:kworker.0 00000100:00100000:0.0:1767319896.008620:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319896.008916:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931328 00000100:00100000:2.0:1767319896.008928:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931328 00000100:00100000:2.0:1767319896.008999:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1389 00000100:00100000:2.0:1767319896.009010:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982931328:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319896.009071:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982931328:12345-0@lo:400:kworker.0 Request processed in 63us (562us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319896.009078:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1389 00000100:00100000:1.0:1767319896.009264:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982931968 00000100:00100000:1.0:1767319896.009309:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982931968 00000100:00080000:0.0:1767319896.009408:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319896.009413:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319896.009444:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398ca80 x1853163982931328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319912 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319896.009457:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982931328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319896.009510:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932096 00000100:00100000:1.0:1767319896.009514:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932096 00000100:00080000:0.0:1767319896.009571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319896.010557:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 865 00000100:00100000:2.0:1767319896.010571:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982931968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319896.010653:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982931968:12345-0@lo:400:kworker.0 Request processed in 86us (4551us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319896.010660:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 865 00000100:00100000:2.0:1767319896.010679:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 866 00000100:00100000:2.0:1767319896.010685:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982932096:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319896.010715:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982932096:12345-0@lo:400:kworker.0 Request processed in 31us (2094us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319896.010721:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 866 00000100:00100000:2.0:1767319896.010764:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982931968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319896.010890:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982932096:0@lo:400:kworker.0 00010000:00010000:2.0:1767319896.196780:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319896.972337:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319896.973020:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319896.973104:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650816 00000100:00100000:1.0:1767319896.973115:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650816 00000100:00100000:1.0:1767319896.973190:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650688 00000100:00100000:1.0:1767319896.973194:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650688 00000100:00100000:1.0:1767319896.973217:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1390 00000100:00100000:1.0:1767319896.973228:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915650816:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319896.973342:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915650816:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 114us (1081us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319896.973351:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1390 00000100:00100000:1.0:1767319896.973371:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1834 00000100:00100000:1.0:1767319896.973375:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915650688:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319896.973397:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319896.973426:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915650688:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (410us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319896.973433:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1834 00000100:00080000:0.0:1767319896.973526:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319897.221149:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319898.244580:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319899.269068:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319899.591960:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319899.591996:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982932480:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319899.592086:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319899.592634:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932480 00000100:00100000:0.0:1767319899.592646:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932480 00000100:00100000:0.0:1767319899.592768:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 938 00000100:00100000:0.0:1767319899.592779:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982932480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319899.592834:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319899.592941:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982932480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 165us (855us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319899.592949:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 938 00000100:00100000:0.0:1767319899.592980:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982932352:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319899.593009:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319899.593054:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932352 00000100:00100000:0.0:1767319899.593069:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932352 00000100:00100000:0.0:1767319899.593104:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939 00000100:00100000:0.0:1767319899.593112:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982932352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319899.593140:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319899.593191:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982932352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (179us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319899.593208:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939 00000100:00100000:0.0:1767319899.593322:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982932352:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319899.593450:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982932480:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767319899.593470:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319899.593474:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767319900.292794:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319901.125004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319901.125017:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767319901.125120:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1767319901.125147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319901.125152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319901.125156:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319901.125166:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5144 00000100:00080000:3.0:1767319901.125172:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319901.125179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319901.125215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163982932736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319901.125317:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982932864:0@lo:400:kworker.0 00000100:00100000:0.0:1767319901.125397:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319901.125480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982932736:0@lo:38:kworker.0 00000100:00080000:3.0:1767319901.125493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319901.125497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319901.125499:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319901.125503:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5144 00000100:00080000:3.0:1767319901.125506:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319901.125511:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319901.125515:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319901.125524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163982932992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319901.125534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319901.125538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319901.125539:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767319901.125542:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982932992:0@lo:38:kworker.0 00000100:00080000:3.0:1767319901.125542:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5144 00000100:00080000:3.0:1767319901.125549:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319901.125553:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319901.125570:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163982933120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319901.125585:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319901.125605:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982933120:0@lo:38:kworker.0 00000100:00100000:1.0:1767319901.125632:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319901.125681:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932864 00000100:00100000:1.0:1767319901.125689:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932864 00000100:00100000:1.0:1767319901.125744:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932736 00000100:00100000:1.0:1767319901.125748:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932736 00000100:00100000:1.0:1767319901.125760:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932992 00000100:00100000:1.0:1767319901.125762:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932992 00000100:00100000:1.0:1767319901.125776:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1391 00000100:00100000:1.0:1767319901.125786:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982932864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319901.126613:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982932608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319901.126664:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319901.126708:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933120 00000100:00100000:2.0:1767319901.126714:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933120 00000100:00100000:2.0:1767319901.126752:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 484 00000100:00100000:2.0:1767319901.126759:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982933120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319901.126785:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319901.126856:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982933120:12345-0@lo:38:kworker.0 Request processed in 88us (1214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319901.126863:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 484 00000100:00080000:2.0:1767319901.126909:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319901.126915:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319901.126918:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982933120:0@lo:38:kworker.0 00000100:00100000:2.0:1767319901.126970:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982932608 00000100:00100000:2.0:1767319901.126975:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982932608 00000100:00100000:2.0:1767319901.127000:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1392 00000100:00100000:2.0:1767319901.127005:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982932608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319901.127047:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982932608:12345-0@lo:400:kworker.0 Request processed in 41us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319901.127053:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1392 00000100:00100000:2.0:1767319901.127068:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1835 00000100:00100000:2.0:1767319901.127071:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319901.127081:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319901.127110:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932736:12345-0@lo:38:kworker.0 Request processed in 37us (1593us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319901.127116:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1835 00000100:00100000:2.0:1767319901.127266:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836 00000100:00100000:2.0:1767319901.127289:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319901.127299:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319901.127327:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982932992:12345-0@lo:38:kworker.0 Request processed in 55us (1741us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319901.127333:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836 00000100:00080000:2.0:1767319901.127350:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319901.127371:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319901.127375:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319901.127379:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982932992:0@lo:38:kworker.0 00000100:00080000:2.0:1767319901.127388:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319901.127392:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319901.127394:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982932736:0@lo:38:kworker.0 00010000:00080000:2.0:1767319901.127421:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017b80 x1853163982932608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319917 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319901.127669:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982932608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319901.127714:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982933248:0@lo:400:kworker.0 00000100:00100000:2.0:1767319901.127747:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319901.127783:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933248 00000100:00100000:2.0:1767319901.127787:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933248 00000100:00100000:2.0:1767319901.127825:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 867 00000100:00100000:2.0:1767319901.127832:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982933248:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767319901.127943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319901.127948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319901.127952:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319901.127957:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5144 00000100:00080000:3.0:1767319901.127960:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319901.127964:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319901.127981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163982933504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319901.128018:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982933504:0@lo:38:kworker.0 00000100:00100000:3.0:1767319901.128050:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319901.128077:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933504 00000100:00100000:3.0:1767319901.128082:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933504 00000100:00100000:3.0:1767319901.128106:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1837 00000100:00100000:3.0:1767319901.128110:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982933504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319901.128123:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319901.130289:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982933248:12345-0@lo:400:kworker.0 Request processed in 2433us (2516us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319901.130303:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 867 00000100:00080000:2.0:1767319901.130356:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319901.130405:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982933248:0@lo:400:kworker.0 00000100:00100000:3.0:1767319901.130678:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982933504:12345-0@lo:38:kworker.0 Request processed in 2563us (2625us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319901.130689:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1837 00000100:00080000:3.0:1767319901.130737:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319901.130744:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319901.130747:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982933504:0@lo:38:kworker.0 00000100:00100000:3.0:1767319901.130774:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982933376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319901.130837:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319901.131881:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933376 00000100:00100000:1.0:1767319901.131889:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982932864:12345-0@lo:400:kworker.0 Request processed in 6098us (6485us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319901.131893:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933376 00000100:00100000:1.0:1767319901.131905:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1391 00000100:00100000:3.0:1767319901.131963:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 868 00000100:00100000:3.0:1767319901.131973:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982933376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319901.132046:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982933376:12345-0@lo:400:kworker.0 Request processed in 75us (1209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319901.132053:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 868 00000100:00100000:3.0:1767319901.132096:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982933376:0@lo:400:kworker.0 00000100:00080000:2.0:1767319901.132594:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:0.0:1767319901.133058:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163982932864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319917 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319901.133072:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982932864:0@lo:400:kworker.0 00010000:00010000:2.0:1767319901.316887:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319902.093868:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319902.093941:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651072 00000100:00100000:2.0:1767319902.093953:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651072 00000100:00100000:2.0:1767319902.094041:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1393 00000100:00100000:2.0:1767319902.094051:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915651072:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319902.094157:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915651072:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 107us (299us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319902.094167:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1393 00000100:00080000:0.0:1767319902.094368:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319902.102506:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319902.102587:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915650944 00000100:00100000:3.0:1767319902.102599:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915650944 00000100:00100000:3.0:1767319902.102653:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1838 00000100:00100000:3.0:1767319902.102661:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915650944:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319902.102690:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319902.102791:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915650944:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 131us (298us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319902.102797:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1838 00010000:00010000:2.0:1767319902.342219:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319903.367257:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319904.388667:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319904.709895:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319904.709927:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982933632:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319904.710007:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319904.711671:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319904.711690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982933760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319904.711756:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319904.715018:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933632 00000100:00100000:2.0:1767319904.715041:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933632 00000100:00100000:0.0:1767319904.715055:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933760 00000100:00100000:0.0:1767319904.715063:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933760 00000100:00100000:2.0:1767319904.715166:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940 00000100:00100000:2.0:1767319904.715179:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982933632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319904.715223:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319904.715370:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982933632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 195us (5364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319904.715380:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940 00000100:00100000:3.0:1767319904.716853:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982933632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319904.731138:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941 00000100:00100000:0.0:1767319904.731158:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982933760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319904.731196:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319904.731718:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982933760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 566us (19959us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319904.731726:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941 00000100:00080000:1.0:1767319904.732651:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319904.732659:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319904.739764:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982933760:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319905.415984:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319906.247600:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319906.247611:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319906.247672:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319906.247675:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319906.247678:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319906.247684:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5149 00000100:00080000:0.0:1767319906.247688:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319906.247694:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319906.247713:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982934016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319906.247733:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319906.247737:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319906.247738:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319906.247742:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5149 00000100:00080000:0.0:1767319906.247744:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319906.247749:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319906.247760:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163982934272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319906.247769:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319906.247772:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319906.247776:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319906.247777:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982933888:0@lo:400:kworker.0 00000100:00080000:0.0:1767319906.247781:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5149 00000100:00080000:0.0:1767319906.247787:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319906.247807:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319906.247821:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc380 x1853163982934400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319906.247859:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319906.247862:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319906.247865:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767319906.247868:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319906.247871:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5149 00000100:00080000:0.0:1767319906.247874:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319906.247877:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319906.247885:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f100 x1853163982934784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319906.247941:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982933888 00000100:00100000:0.0:1767319906.247948:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982933888 00000100:00100000:2.0:1767319906.247957:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319906.247966:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982934656:0@lo:400:kworker.0 00000100:00100000:0.0:1767319906.247995:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1394 00000100:00100000:0.0:1767319906.248011:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982933888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319906.248013:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319906.248082:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982933888:12345-0@lo:400:kworker.0 Request processed in 77us (217us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319906.248090:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1394 00010000:00080000:2.0:1767319906.248128:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc000 x1853163982933888/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319922 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319906.248143:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982933888:0@lo:400:kworker.0 00000100:00100000:3.0:1767319906.248252:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982934144:0@lo:400:kworker.0 00000100:00100000:3.0:1767319906.248312:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319906.248330:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319906.248334:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982934528:0@lo:400:kworker.0 00000100:00100000:3.0:1767319906.248350:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319906.248384:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982934016:0@lo:38:kworker.0 00000100:00100000:3.0:1767319906.248400:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319906.248418:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982934272:0@lo:38:kworker.0 00000100:00100000:3.0:1767319906.248433:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319906.248437:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934656 00000100:00100000:2.0:1767319906.248443:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934656 00000100:00100000:3.0:1767319906.248448:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982934400:0@lo:38:kworker.0 00000100:00100000:3.0:1767319906.248465:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319906.248485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982934784:0@lo:38:kworker.0 00000100:00100000:2.0:1767319906.248492:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934528 00000100:00100000:2.0:1767319906.248498:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934528 00000100:00100000:3.0:1767319906.248502:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319906.248526:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934144 00000100:00100000:3.0:1767319906.248529:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934144 00000100:00100000:2.0:1767319906.248534:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 869 00000100:00100000:2.0:1767319906.248544:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982934656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319906.248583:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982934656:12345-0@lo:400:kworker.0 Request processed in 42us (575us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319906.248590:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 869 00000100:00100000:2.0:1767319906.248602:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 870 00000100:00100000:2.0:1767319906.248608:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982934528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319906.248622:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934016 00000100:00100000:3.0:1767319906.248626:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934016 00000100:00100000:3.0:1767319906.248636:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934272 00000100:00100000:3.0:1767319906.248637:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934272 00000100:00100000:3.0:1767319906.248644:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934784 00000100:00100000:3.0:1767319906.248646:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934784 00000100:00100000:3.0:1767319906.248652:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1395 00000100:00100000:2.0:1767319906.248658:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982934528:12345-0@lo:400:kworker.0 Request processed in 37us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319906.248658:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982934144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319906.248671:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 870 00000100:00100000:3.0:1767319906.248692:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982934144:12345-0@lo:400:kworker.0 Request processed in 33us (378us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319906.248698:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1839 00000100:00100000:2.0:1767319906.248704:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982934016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319906.248722:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319906.248759:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982934016:12345-0@lo:38:kworker.0 Request processed in 53us (356us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767319906.248802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319906.248806:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319906.248807:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319906.248832:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934400 00000100:00100000:1.0:1767319906.248844:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934400 00000100:00100000:1.0:1767319906.248862:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 485 00000100:00100000:1.0:1767319906.248866:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982934400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319906.248873:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319906.248901:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982934400:12345-0@lo:38:kworker.0 Request processed in 34us (436us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319906.248905:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 485 00000100:00100000:3.0:1767319906.257213:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1395 00000100:00100000:3.0:1767319906.257291:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840 00000100:00100000:3.0:1767319906.257299:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982934272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319906.257323:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319906.257409:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982934272:12345-0@lo:38:kworker.0 Request processed in 111us (8976us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319906.257414:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840 00000100:00100000:3.0:1767319906.257426:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1841 00000100:00100000:3.0:1767319906.257430:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982934784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319906.257436:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319906.257451:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982934784:12345-0@lo:38:kworker.0 Request processed in 21us (8951us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319906.257454:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1841 00000100:00100000:3.0:1767319906.257497:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982934528:0@lo:400:kworker.0 00010000:00080000:3.0:1767319906.257508:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fdc00 x1853163982934144/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319922 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319906.257515:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982934144:0@lo:400:kworker.0 00000100:00080000:3.0:1767319906.257785:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319906.257791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319906.257795:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982934016:0@lo:38:kworker.0 00000100:00100000:0.0:1767319906.262762:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982934656:0@lo:400:kworker.0 00000100:00100000:2.0:1767319906.267267:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1839 00000100:00080000:3.0:1767319906.268719:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319906.268731:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319906.268737:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982934272:0@lo:38:kworker.0 00000100:00080000:3.0:1767319906.268755:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319906.268760:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319906.268762:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982934400:0@lo:38:kworker.0 00000100:00080000:3.0:1767319906.268773:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319906.268776:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319906.268779:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982934784:0@lo:38:kworker.0 00010000:00010000:1.0:1767319906.436653:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319907.211786:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319907.211820:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319907.211956:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651328 00000100:00100000:3.0:1767319907.211968:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651328 00000100:00100000:3.0:1767319907.212051:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651200 00000100:00100000:3.0:1767319907.212054:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651200 00000100:00100000:3.0:1767319907.212072:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1396 00000100:00100000:3.0:1767319907.212111:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915651328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319907.212193:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915651328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 113us (418us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319907.212200:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1396 00000100:00100000:3.0:1767319907.212219:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1842 00000100:00100000:3.0:1767319907.212223:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915651200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319907.212245:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319907.212265:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915651200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (444us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319907.212328:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1842 00000100:00080000:1.0:1767319907.212948:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319907.460523:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319908.486655:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319909.514801:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319909.830871:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319909.830897:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982934912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319909.831008:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319909.831899:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982934912 00000100:00100000:2.0:1767319909.831914:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982934912 00000100:00100000:2.0:1767319909.832010:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 942 00000100:00100000:2.0:1767319909.832020:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982934912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319909.832073:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319909.832153:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982934912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (1145us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319909.832160:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 942 00000100:00100000:3.0:1767319909.832409:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982935040:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319909.832454:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319909.832618:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935040 00000100:00100000:0.0:1767319909.832623:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935040 00000100:00100000:0.0:1767319909.832672:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 943 00000100:00100000:0.0:1767319909.832683:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982935040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319909.832706:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319909.832745:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982935040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (291us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319909.832762:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 943 00000100:00080000:1.0:1767319909.833000:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319909.833006:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319909.833062:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982934912:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319909.833159:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982935040:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319910.532518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319911.365353:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319911.365365:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767319911.365473:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767319911.366375:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319911.366381:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319911.366384:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319911.366392:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5154 00000100:00080000:2.0:1767319911.366396:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319911.366403:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319911.366432:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163982935296/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319911.366479:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319911.366482:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319911.366484:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319911.366488:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5154 00000100:00080000:2.0:1767319911.366490:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319911.366495:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319911.366506:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163982935552/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319911.366514:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319911.366518:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319911.366519:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319911.366522:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5154 00000100:00080000:2.0:1767319911.366525:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319911.366528:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319911.366543:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc380 x1853163982935680/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319911.366712:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982935808:0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.366781:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319911.366838:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319911.366843:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982935424:0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.366865:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319911.366895:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935424 00000100:00100000:0.0:1767319911.366902:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935424 00000100:00100000:0.0:1767319911.366949:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1397 00000100:00100000:0.0:1767319911.366957:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982935424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.367017:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982935424:12345-0@lo:400:kworker.0 Request processed in 49us (138us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319911.367024:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1397 00010000:00080000:0.0:1767319911.367063:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fdc00 x1853163982935424/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319927 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319911.367076:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982935424:0@lo:400:kworker.0 00000100:00100000:1.0:1767319911.368406:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935808 00000100:00100000:1.0:1767319911.368423:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935808 00000100:00100000:1.0:1767319911.368503:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 871 00000100:00100000:1.0:1767319911.368514:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982935808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319911.368613:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982935808:12345-0@lo:400:kworker.0 Request processed in 102us (1832us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319911.368621:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 871 00000100:00080000:1.0:1767319911.368647:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319911.368652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319911.368675:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982935296:0@lo:38:kworker.0 00000100:00100000:1.0:1767319911.368719:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319911.368754:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982935552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319911.368776:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319911.368797:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982935680:0@lo:38:kworker.0 00000100:00100000:1.0:1767319911.368821:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319911.368881:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982935808:0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.376089:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935680 00000100:00100000:0.0:1767319911.376109:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935680 00000100:00100000:0.0:1767319911.376164:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 486 00000100:00100000:0.0:1767319911.376173:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982935680:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319911.376198:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319911.376327:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982935680:12345-0@lo:38:kworker.0 Request processed in 154us (7504us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319911.376335:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 486 00000100:00100000:0.0:1767319911.376361:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935296 00000100:00100000:0.0:1767319911.376366:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935296 00000100:00100000:0.0:1767319911.376384:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935552 00000100:00100000:0.0:1767319911.376387:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935552 00000100:00100000:0.0:1767319911.376401:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1843 00000100:00100000:0.0:1767319911.376405:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982935296:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319911.376415:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319911.376440:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982935296:12345-0@lo:38:kworker.0 Request processed in 34us (7721us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319911.376446:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1843 00000100:00100000:0.0:1767319911.376457:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1844 00000100:00100000:0.0:1767319911.376460:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982935552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319911.376468:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319911.376490:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982935552:12345-0@lo:38:kworker.0 Request processed in 29us (7714us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319911.376495:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1844 00000100:00080000:0.0:1767319911.376566:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319911.376572:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319911.376576:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982935296:0@lo:38:kworker.0 00000100:00080000:0.0:1767319911.376587:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319911.376590:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319911.376593:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982935552:0@lo:38:kworker.0 00000100:00080000:0.0:1767319911.376603:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319911.376606:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319911.376608:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982935680:0@lo:38:kworker.0 00000100:00100000:3.0:1767319911.376817:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982935936:0@lo:400:kworker.0 00000100:00100000:3.0:1767319911.376863:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319911.376897:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982935168:0@lo:400:kworker.0 00000100:00100000:3.0:1767319911.376919:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319911.376948:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935168 00000100:00100000:3.0:1767319911.376952:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935168 00000100:00100000:3.0:1767319911.376999:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1398 00000100:00100000:3.0:1767319911.377016:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982935168:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319911.377067:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982935168:12345-0@lo:400:kworker.0 Request processed in 63us (147us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319911.377075:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1398 00010000:00080000:3.0:1767319911.377112:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe680 x1853163982935168/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319927 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319911.377125:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982935168:0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.377310:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982935936 00000100:00100000:0.0:1767319911.377315:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982935936 00000100:00100000:0.0:1767319911.377358:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 872 00000100:00100000:0.0:1767319911.377365:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982935936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319911.377409:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982935936:12345-0@lo:400:kworker.0 Request processed in 45us (546us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319911.377415:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 872 00000100:00080000:2.0:1767319911.377513:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319911.377531:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319911.377534:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319911.377540:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5154 00000100:00080000:2.0:1767319911.377545:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319911.377551:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:1.0:1767319911.377581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319911.377585:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e245f80 x1853163982936064/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319911.377588:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319911.377619:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982936064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319911.377658:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319911.377742:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936064 00000100:00100000:2.0:1767319911.377747:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936064 00000100:00100000:2.0:1767319911.377779:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1845 00000100:00100000:2.0:1767319911.377784:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982936064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319911.377802:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319911.377803:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982935936:0@lo:400:kworker.0 00000100:00100000:2.0:1767319911.377850:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982936064:12345-0@lo:38:kworker.0 Request processed in 64us (191us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319911.377857:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1845 00000100:00080000:1.0:1767319911.377969:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319911.377974:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319911.377977:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982936064:0@lo:38:kworker.0 00010000:00010000:2.0:1767319911.557627:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319912.337567:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319912.340250:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651584 00000100:00100000:1.0:1767319912.340282:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651584 00000100:00100000:1.0:1767319912.340387:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1399 00000100:00080000:0.0:1767319912.340481:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319912.340523:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915651584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319912.340862:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915651584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 342us (3307us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319912.340869:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1399 00000100:00100000:1.0:1767319912.352395:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767319912.352524:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915651712 offset 0 length 520: 4 00000100:00100000:2.0:1767319912.352632:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651456 00000100:00100000:2.0:1767319912.352647:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651456 00000100:00100000:2.0:1767319912.352719:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1846 00000100:00100000:2.0:1767319912.352728:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915651456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319912.352755:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319912.352831:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915651456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (448us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319912.352838:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1846 00010000:00010000:1.0:1767319912.582405:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319913.605069:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319914.628740:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319914.948789:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319914.948820:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982936192:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319914.948891:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319914.949052:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936192 00000100:00100000:2.0:1767319914.949061:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936192 00000100:00100000:2.0:1767319914.949135:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1767319914.949152:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982936192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319914.949182:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319914.949232:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982936192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (340us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319914.949240:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944 00000100:00080000:1.0:1767319914.949296:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319914.949661:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982936192:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319914.950053:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982936320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319914.950096:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319914.950176:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936320 00000100:00100000:3.0:1767319914.950182:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936320 00000100:00100000:3.0:1767319914.950230:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945 00000100:00100000:3.0:1767319914.950238:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982936320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319914.950259:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767319914.950361:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319914.955157:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982936320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4897us (5038us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319914.955176:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945 00000100:00100000:1.0:1767319914.959837:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982936320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319915.462670:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319890/real 1767319890] req@ffff9346100fd180 x1853163982930944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319915 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319915.462709:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100fd180 x1853163982930944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319915 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767319915.462732:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319915.462764:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767319915.462769:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982930944:0@lo:250:kworker.0 00000100:00080000:0.0:1767319915.463104:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.463111:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319915.463114:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.463119:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5139 00000100:00080000:0.0:1767319915.463123:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.463130:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319915.463167:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469dc00 x1853163982936448/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.463236:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.463241:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.463243:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.463247:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5159 00000100:00080000:0.0:1767319915.463250:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.463253:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319915.463318:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982936704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.463343:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.463346:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.463348:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.463351:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5159 00000100:00080000:0.0:1767319915.463354:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.463358:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319915.463371:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469fb80 x1853163982936960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319915.463569:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982936832:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.463644:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.463715:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319915.463724:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982936576:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.463750:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319915.463846:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936832 00000100:00100000:1.0:1767319915.463855:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936832 00000100:00100000:1.0:1767319915.463947:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936576 00000100:00100000:1.0:1767319915.463951:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936576 00000100:00100000:1.0:1767319915.463978:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1400 00000100:00100000:1.0:1767319915.463988:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982936832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319915.464049:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982936832:12345-0@lo:400:kworker.0 Request processed in 62us (404us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319915.464057:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1400 00000100:00100000:1.0:1767319915.464077:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1401 00000100:00100000:1.0:1767319915.464083:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982936576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319915.464112:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982936576:12345-0@lo:400:kworker.0 Request processed in 30us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319915.464119:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1401 00000100:00100000:1.0:1767319915.464155:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982936448:0@lo:250:kworker.0 00000400:00000100:1.0:1767319915.464176:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982936448 offset 0 length 520: 4 00000100:00100000:1.0:1767319915.464195:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982936704:0@lo:38:kworker.0 00000100:00100000:1.0:1767319915.464216:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319915.464243:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936704 00000100:00100000:1.0:1767319915.464247:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936704 00000100:00100000:1.0:1767319915.464266:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1847 00000100:00100000:1.0:1767319915.464292:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982936704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319915.464327:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319915.464362:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982936704:12345-0@lo:38:kworker.0 Request processed in 70us (145us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319915.464369:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1847 00000100:00080000:1.0:1767319915.464416:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319915.464421:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319915.464424:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982936704:0@lo:38:kworker.0 00000100:00080000:3.0:1767319915.467617:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319915.467686:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ca80 x1853163982936832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319931 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319915.467705:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982936832:0@lo:400:kworker.0 00010000:00080000:3.0:1767319915.467759:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469e680 x1853163982936576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319931 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319915.467772:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982936576:0@lo:400:kworker.0 00000100:00080000:0.0:1767319915.468053:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.468058:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.468063:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.468068:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5159 00000100:00080000:0.0:1767319915.468081:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.468088:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319915.468134:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d180 x1853163982937088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319915.468185:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319915.468189:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319915.468191:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319915.468195:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5159 00000100:00080000:0.0:1767319915.468198:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319915.468201:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319915.468214:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e300 x1853163982937472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319915.468466:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982937216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.468571:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319915.468644:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982937344:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.468668:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.468836:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982936960:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.468884:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.468912:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982937088:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.468944:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.468964:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982937472:0@lo:38:kworker.0 00000100:00100000:2.0:1767319915.468985:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319915.469016:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937088 00000100:00100000:2.0:1767319915.469023:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937088 00000100:00100000:2.0:1767319915.469061:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 487 00000100:00100000:2.0:1767319915.469070:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982937088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319915.469092:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319915.469135:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982937088:12345-0@lo:38:kworker.0 Request processed in 64us (189us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319915.469142:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 487 00000100:00080000:2.0:1767319915.469180:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.469185:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.469189:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982937088:0@lo:38:kworker.0 00000100:00100000:3.0:1767319915.469517:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982936960 00000100:00100000:3.0:1767319915.469522:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982936960 00000100:00100000:3.0:1767319915.469553:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937472 00000100:00100000:3.0:1767319915.469557:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937472 00000100:00100000:3.0:1767319915.469575:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848 00000100:00100000:3.0:1767319915.469579:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982936960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319915.469591:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319915.469638:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982936960:12345-0@lo:38:kworker.0 Request processed in 57us (754us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319915.469644:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848 00000100:00100000:3.0:1767319915.469656:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1849 00000100:00100000:3.0:1767319915.469660:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982937472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319915.469668:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319915.469690:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982937472:12345-0@lo:38:kworker.0 Request processed in 28us (704us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319915.469696:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1849 00000100:00100000:1.0:1767319915.469797:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937216 00000100:00100000:1.0:1767319915.469803:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937216 00000100:00100000:1.0:1767319915.469851:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937344 00000100:00100000:1.0:1767319915.469855:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937344 00000100:00100000:1.0:1767319915.469879:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 873 00000100:00100000:1.0:1767319915.469885:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982937216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319915.469956:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982937216:12345-0@lo:400:kworker.0 Request processed in 71us (1386us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319915.469962:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 873 00000100:00100000:1.0:1767319915.469974:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767319915.469980:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982937344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319915.470010:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982937344:12345-0@lo:400:kworker.0 Request processed in 31us (1342us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319915.470016:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 874 00000100:00100000:1.0:1767319915.470054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982937216:0@lo:400:kworker.0 00000100:00100000:3.0:1767319915.470146:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982937344:0@lo:400:kworker.0 00000100:00080000:2.0:1767319915.471812:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.471825:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.471829:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982936960:0@lo:38:kworker.0 00000100:00080000:2.0:1767319915.471842:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319915.471846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319915.471849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982937472:0@lo:38:kworker.0 00010000:00010000:1.0:1767319915.660035:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319916.677688:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319917.452490:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319917.452612:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651840 00000100:00100000:2.0:1767319917.452623:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651840 00000100:00100000:2.0:1767319917.452675:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1850 00000100:00100000:2.0:1767319917.452684:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915651840:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319917.452707:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319917.452780:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915651840:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 94us (298us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319917.452788:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1850 00000100:00100000:3.0:1767319917.453321:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319917.453370:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915651968 00000100:00100000:1.0:1767319917.453377:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915651968 00000100:00100000:1.0:1767319917.453445:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1402 00000100:00100000:1.0:1767319917.453453:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915651968:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319917.453505:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915651968:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 52us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319917.453513:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1402 00000100:00080000:0.0:1767319917.453612:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319917.705340:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319918.730824:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319919.752468:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319920.068454:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982937600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319920.068526:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.068873:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982937728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319920.068907:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319920.068996:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937600 00000100:00100000:2.0:1767319920.069006:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937600 00000100:00100000:2.0:1767319920.069079:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937728 00000100:00100000:2.0:1767319920.069083:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937728 00000100:00100000:2.0:1767319920.069106:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 946 00000100:00100000:2.0:1767319920.069114:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982937600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319920.069146:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319920.069193:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982937600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (667us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319920.069201:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 946 00000100:00100000:2.0:1767319920.069230:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947 00000100:00100000:2.0:1767319920.069237:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982937728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319920.069254:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319920.069312:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982937728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (405us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319920.069319:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947 00000100:00100000:2.0:1767319920.069360:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982937600:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767319920.069494:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767319920.069499:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319920.069523:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982937728:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319920.580683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319920.580695:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319920.580761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319920.580766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319920.580769:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319920.580778:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5164 00000100:00080000:1.0:1767319920.580782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319920.580788:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319920.580812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa680 x1853163982937984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319920.580853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319920.580856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319920.580858:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319920.580861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5164 00000100:00080000:1.0:1767319920.580863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319920.580867:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319920.580878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8a80 x1853163982938240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319920.580887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319920.580890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319920.580892:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319920.580895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5164 00000100:00080000:1.0:1767319920.580897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319920.580900:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319920.580914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0abb80 x1853163982938368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319920.580932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319920.580936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319920.580938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319920.580941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5164 00000100:00080000:1.0:1767319920.580943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319920.580947:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319920.580956:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9880 x1853163982938752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319920.581135:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982937856:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581199:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.581247:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319920.581253:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982938624:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581301:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.581327:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982938112:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581350:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.581367:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319920.581372:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982938496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581391:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.581419:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938624 00000100:00100000:3.0:1767319920.581426:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938624 00000100:00100000:3.0:1767319920.581469:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938496 00000100:00100000:3.0:1767319920.581472:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938496 00000100:00100000:3.0:1767319920.581493:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 875 00000100:00100000:3.0:1767319920.581501:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982938624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581549:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982938624:12345-0@lo:400:kworker.0 Request processed in 48us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319920.581555:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 875 00000100:00100000:3.0:1767319920.581571:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 876 00000100:00100000:3.0:1767319920.581577:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982938496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581606:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982938496:12345-0@lo:400:kworker.0 Request processed in 30us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319920.581611:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 876 00000100:00100000:3.0:1767319920.581652:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982938496:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.581692:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982938624:0@lo:400:kworker.0 00000100:00100000:1.0:1767319920.581758:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937856 00000100:00100000:1.0:1767319920.581762:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937856 00000100:00100000:1.0:1767319920.581801:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938112 00000100:00100000:1.0:1767319920.581804:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938112 00000100:00100000:1.0:1767319920.581820:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1403 00000100:00100000:1.0:1767319920.581827:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982937856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319920.581874:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982937856:12345-0@lo:400:kworker.0 Request processed in 48us (675us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319920.581881:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1403 00000100:00100000:1.0:1767319920.581894:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1404 00000100:00100000:1.0:1767319920.581900:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982938112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319920.581926:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982938112:12345-0@lo:400:kworker.0 Request processed in 27us (576us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319920.581932:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1404 00000100:00080000:3.0:1767319920.583774:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767319920.583829:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a8e00 x1853163982938112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319936 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319920.583856:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982938112:0@lo:400:kworker.0 00010000:00080000:3.0:1767319920.583896:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9c00 x1853163982937856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319936 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319920.583905:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982937856:0@lo:400:kworker.0 00000100:00100000:3.0:1767319920.583951:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982937984:0@lo:38:kworker.0 00000100:00100000:3.0:1767319920.584158:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.584249:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982938240:0@lo:38:kworker.0 00000100:00100000:3.0:1767319920.584372:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.585039:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982938368:0@lo:38:kworker.0 00000100:00100000:3.0:1767319920.585073:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319920.585101:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982938752:0@lo:38:kworker.0 00000100:00100000:3.0:1767319920.585122:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319920.585170:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938368 00000100:00100000:2.0:1767319920.585178:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938368 00000100:00100000:2.0:1767319920.585215:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 488 00000100:00100000:2.0:1767319920.585223:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982938368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319920.585257:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319920.585317:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982938368:12345-0@lo:38:kworker.0 Request processed in 94us (243us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319920.585325:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 488 00000100:00100000:0.0:1767319920.585587:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982937984 00000100:00100000:0.0:1767319920.585592:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982937984 00000100:00100000:1.0:1767319920.585636:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938240 00000100:00100000:1.0:1767319920.585640:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938240 00000100:00100000:1.0:1767319920.585677:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1851 00000100:00100000:1.0:1767319920.585683:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982937984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319920.585694:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319920.585729:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982937984:12345-0@lo:38:kworker.0 Request processed in 46us (1574us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319920.585734:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1851 00000100:00100000:1.0:1767319920.585747:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852 00000100:00100000:1.0:1767319920.585750:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982938240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319920.585759:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319920.585780:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982938240:12345-0@lo:38:kworker.0 Request processed in 28us (1409us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319920.585785:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852 00000100:00080000:3.0:1767319920.586056:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319920.586061:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319920.586066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982937984:0@lo:38:kworker.0 00000100:00080000:3.0:1767319920.586075:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319920.586078:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319920.586081:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982938240:0@lo:38:kworker.0 00000100:00080000:3.0:1767319920.586089:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319920.586092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319920.586095:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982938368:0@lo:38:kworker.0 00000100:00100000:2.0:1767319920.586146:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938752 00000100:00100000:2.0:1767319920.586150:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938752 00000100:00100000:2.0:1767319920.586175:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1853 00000100:00100000:2.0:1767319920.586180:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982938752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319920.586189:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319920.586237:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982938752:12345-0@lo:38:kworker.0 Request processed in 48us (1106us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319920.586243:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1853 00000100:00080000:3.0:1767319920.586507:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319920.586512:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319920.586515:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982938752:0@lo:38:kworker.0 00010000:00010000:2.0:1767319920.772511:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319921.796430:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319922.572394:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319922.572456:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652224 00000100:00100000:0.0:1767319922.572467:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652224 00000100:00100000:0.0:1767319922.572548:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1405 00000100:00100000:0.0:1767319922.572558:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915652224:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319922.572639:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915652224:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (258us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319922.572647:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1405 00000100:00080000:3.0:1767319922.572766:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319922.578995:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319922.579067:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652096 00000100:00100000:1.0:1767319922.579436:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652096 00000100:00100000:1.0:1767319922.579687:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1854 00000100:00100000:1.0:1767319922.579696:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915652096:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319922.579719:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319922.579793:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915652096:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 96us (806us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319922.579814:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1854 00010000:00010000:2.0:1767319922.820732:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319923.845027:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319924.871073:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319925.188759:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319925.188784:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982938880:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319925.188855:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.188996:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982938880 00000100:00100000:1.0:1767319925.189004:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982938880 00000100:00100000:1.0:1767319925.189077:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948 00000100:00100000:1.0:1767319925.189090:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982938880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319925.189125:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319925.189196:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982938880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (341us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319925.189208:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948 00000100:00100000:1.0:1767319925.189310:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982938880:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319925.189365:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982939008:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319925.189398:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.189648:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939008 00000100:00100000:1.0:1767319925.189653:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939008 00000100:00100000:1.0:1767319925.189689:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 949 00000100:00100000:1.0:1767319925.189698:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982939008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319925.189716:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319925.189768:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982939008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319925.189776:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 949 00000100:00100000:1.0:1767319925.189820:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982939008:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767319925.189858:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319925.189863:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767319925.701556:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319925.701566:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319925.701626:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319925.701630:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319925.701633:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319925.701641:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5169 00000100:00080000:2.0:1767319925.701645:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767319925.701650:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767319925.701652:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319925.701667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982939136:0@lo:400:kworker.0 00000100:00100000:2.0:1767319925.701673:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163982939264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319925.701721:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319925.701725:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319925.701727:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319925.701730:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5169 00000100:00080000:2.0:1767319925.701733:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319925.701737:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319925.701758:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93100 x1853163982939520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319925.701772:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319925.701775:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319925.701776:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319925.701779:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5169 00000100:00080000:2.0:1767319925.701781:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319925.701785:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319925.701796:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92680 x1853163982939648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319925.701816:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319925.701819:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319925.701821:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319925.701824:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5169 00000100:00080000:2.0:1767319925.701826:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319925.701829:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319925.701837:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90e00 x1853163982940032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319925.701860:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982939392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.701881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.701888:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982939776:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.701934:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982939904:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.701958:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.701988:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939136 00000100:00100000:1.0:1767319925.701996:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939136 00000100:00100000:0.0:1767319925.702114:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939904 00000100:00100000:0.0:1767319925.702118:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939904 00000100:00100000:0.0:1767319925.702164:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 877 00000100:00100000:0.0:1767319925.702174:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982939904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319925.702226:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982939904:12345-0@lo:400:kworker.0 Request processed in 54us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319925.702234:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 877 00000100:00100000:0.0:1767319925.702256:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982939264:0@lo:38:kworker.0 00000100:00100000:1.0:1767319925.702265:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1406 00000100:00100000:1.0:1767319925.702291:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982939136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319925.702313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.702320:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982939136:12345-0@lo:400:kworker.0 Request processed in 50us (441us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319925.702327:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1406 00000100:00100000:0.0:1767319925.702331:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982939520:0@lo:38:kworker.0 00000100:00100000:1.0:1767319925.702339:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939264 00000100:00100000:1.0:1767319925.702342:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939264 00000100:00100000:0.0:1767319925.702349:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.702410:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319925.713236:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982939648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319925.713315:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319925.713409:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939520 00000100:00100000:1.0:1767319925.713419:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939520 00000100:00100000:3.0:1767319925.713720:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939776 00000100:00100000:3.0:1767319925.713725:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939776 00000100:00100000:3.0:1767319925.713758:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1855 00000100:00100000:3.0:1767319925.713767:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982939264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319925.713788:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319925.713831:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982939264:12345-0@lo:38:kworker.0 Request processed in 64us (11517us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319925.713838:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1855 00000100:00100000:3.0:1767319925.713860:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1856 00000100:00100000:3.0:1767319925.713864:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982939520:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319925.713873:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319925.713896:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982939520:12345-0@lo:38:kworker.0 Request processed in 30us (11546us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319925.713901:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1856 00000100:00100000:3.0:1767319925.713962:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 878 00000100:00100000:3.0:1767319925.713969:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982939776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.714024:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982939776:12345-0@lo:400:kworker.0 Request processed in 56us (11614us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319925.714030:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 878 00000100:00100000:3.0:1767319925.714093:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982939904:0@lo:400:kworker.0 00010000:00080000:3.0:1767319925.714108:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e246300 x1853163982939136/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319941 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319925.714118:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982939136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319925.714178:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982939776:0@lo:400:kworker.0 00000100:00080000:3.0:1767319925.714210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319925.714225:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319925.714292:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939392 00000100:00100000:2.0:1767319925.714297:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939392 00000100:00080000:2.0:1767319925.714317:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319925.714345:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1407 00000100:00100000:2.0:1767319925.714352:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982939392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319925.714388:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982939392:12345-0@lo:400:kworker.0 Request processed in 37us (163us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319925.714394:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1407 00000100:00100000:2.0:1767319925.714441:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982939648 00000100:00100000:2.0:1767319925.714445:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982939648 00000100:00100000:2.0:1767319925.714466:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 489 00000100:00100000:2.0:1767319925.714471:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982939648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319925.714481:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319925.714507:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982939648:12345-0@lo:38:kworker.0 Request processed in 34us (1192us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319925.714513:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 489 00000100:00080000:0.0:1767319925.719413:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319925.719428:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319925.719432:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982939648:0@lo:38:kworker.0 00000100:00100000:0.0:1767319925.719445:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982940032:0@lo:38:kworker.0 00000100:00100000:0.0:1767319925.719510:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319925.719578:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319925.719582:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319925.719586:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982939264:0@lo:38:kworker.0 00000100:00080000:0.0:1767319925.719596:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319925.719599:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319925.719603:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982939520:0@lo:38:kworker.0 00010000:00080000:3.0:1767319925.722840:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d90380 x1853163982939392/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319941 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319925.722861:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982939392:0@lo:400:kworker.0 00000100:00100000:1.0:1767319925.724198:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940032 00000100:00100000:1.0:1767319925.724212:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940032 00000100:00100000:1.0:1767319925.724266:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1857 00000100:00100000:1.0:1767319925.724288:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940032:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319925.724309:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319925.725190:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940032:12345-0@lo:38:kworker.0 Request processed in 898us (5675us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319925.725200:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1857 00000100:00080000:0.0:1767319925.725701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319925.725708:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319925.725711:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982940032:0@lo:38:kworker.0 00010000:00010000:2.0:1767319925.892968:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319926.916508:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319927.695043:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319927.695141:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652480 00000100:00100000:3.0:1767319927.695164:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652480 00000100:00100000:3.0:1767319927.695250:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1408 00000100:00100000:3.0:1767319927.695261:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915652480:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319927.695403:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915652480:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 143us (375us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319927.695411:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1408 00000100:00080000:2.0:1767319927.695957:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319927.698456:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319927.698573:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652352 00000100:00100000:1.0:1767319927.698583:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652352 00000100:00100000:1.0:1767319927.698630:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1858 00000100:00100000:1.0:1767319927.698638:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915652352:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319927.698661:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319927.698746:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915652352:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 108us (300us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319927.698755:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1858 00010000:00010000:1.0:1767319927.940668:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319928.965155:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319929.989261:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319930.308666:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767319930.308694:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982940160:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319930.308794:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319930.308985:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940160 00000100:00100000:1.0:1767319930.308993:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940160 00000100:00100000:1.0:1767319930.309054:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 950 00000100:00100000:1.0:1767319930.309063:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982940160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319930.309099:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319930.309143:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982940160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 81us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319930.309150:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 950 00000100:00100000:1.0:1767319930.309320:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982940288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319930.309349:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319930.309429:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940288 00000100:00100000:0.0:1767319930.309433:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940288 00000100:00100000:0.0:1767319930.309467:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951 00000100:00100000:0.0:1767319930.309475:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982940288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319930.309491:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319930.309525:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982940288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319930.309532:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951 00000100:00100000:0.0:1767319930.309572:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982940288:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767319930.309833:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767319930.309838:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319930.309877:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982940160:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767319930.820765:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319930.820777:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319930.820847:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319930.820851:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319930.820855:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319930.820863:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5174 00000100:00080000:0.0:1767319930.820869:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319930.820875:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319930.820896:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982940544/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319930.820926:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319930.820930:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319930.820931:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319930.820934:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5174 00000100:00080000:0.0:1767319930.820937:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319930.820940:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319930.820950:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ea00 x1853163982940800/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319930.820957:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319930.820960:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319930.820961:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319930.820964:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5174 00000100:00080000:0.0:1767319930.820966:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319930.820969:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319930.820978:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f100 x1853163982940928/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319930.821011:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319930.821014:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319930.821016:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319930.821019:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5174 00000100:00080000:0.0:1767319930.821021:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319930.821024:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319930.821034:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d500 x1853163982941312/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319930.821072:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982941184:0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.821360:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982940544:0@lo:38:kworker.0 00000100:00100000:3.0:1767319930.821445:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982940416:0@lo:400:kworker.0 00000100:00100000:1.0:1767319930.823698:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982940672:0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.828860:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.828952:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982940800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319930.828997:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.829034:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982940928:0@lo:38:kworker.0 00000100:00100000:2.0:1767319930.829061:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.829086:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982941312:0@lo:38:kworker.0 00000100:00100000:2.0:1767319930.829109:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.829185:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940544 00000100:00100000:2.0:1767319930.829194:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940544 00000100:00100000:2.0:1767319930.829233:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940800 00000100:00100000:2.0:1767319930.829236:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940800 00000100:00100000:2.0:1767319930.829252:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941312 00000100:00100000:2.0:1767319930.829254:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941312 00000100:00100000:2.0:1767319930.829287:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1859 00000100:00100000:2.0:1767319930.829297:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319930.829319:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319930.829366:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940544:12345-0@lo:38:kworker.0 Request processed in 68us (515us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319930.829373:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1859 00000100:00100000:2.0:1767319930.829389:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860 00000100:00100000:2.0:1767319930.829393:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319930.829401:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319930.829425:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982940800:12345-0@lo:38:kworker.0 Request processed in 30us (427us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319930.829431:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860 00000100:00100000:2.0:1767319930.829441:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1861 00000100:00100000:2.0:1767319930.829445:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982941312:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319930.829454:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319930.829476:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982941312:12345-0@lo:38:kworker.0 Request processed in 29us (365us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319930.829481:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1861 00000100:00100000:2.0:1767319930.829499:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940928 00000100:00100000:2.0:1767319930.829502:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940928 00000100:00100000:2.0:1767319930.829522:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 490 00000100:00100000:2.0:1767319930.829527:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982940928:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319930.829535:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319930.829560:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982940928:12345-0@lo:38:kworker.0 Request processed in 33us (498us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319930.829566:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 490 00000100:00080000:2.0:1767319930.829616:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319930.829622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319930.829625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982940544:0@lo:38:kworker.0 00000100:00080000:2.0:1767319930.829633:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319930.829636:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319930.829639:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982940800:0@lo:38:kworker.0 00000100:00080000:2.0:1767319930.829659:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319930.829662:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319930.829665:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982940928:0@lo:38:kworker.0 00000100:00080000:2.0:1767319930.829672:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319930.829675:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319930.829677:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982941312:0@lo:38:kworker.0 00000100:00100000:2.0:1767319930.829736:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982941056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.829766:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.829799:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941056 00000100:00100000:2.0:1767319930.829803:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941056 00000100:00100000:2.0:1767319930.829837:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767319930.829843:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982941056:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.829889:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982941056:12345-0@lo:400:kworker.0 Request processed in 45us (121us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319930.829895:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 879 00000100:00100000:2.0:1767319930.829923:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982941056:0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.830863:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319930.830967:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940416 00000100:00100000:3.0:1767319930.830976:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940416 00000100:00100000:3.0:1767319930.831032:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1409 00000100:00100000:3.0:1767319930.831041:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982940416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319930.831094:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982940416:12345-0@lo:400:kworker.0 Request processed in 55us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319930.831101:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1409 00010000:00080000:3.0:1767319930.831155:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ca80 x1853163982940416/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319946 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319930.831169:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982940416:0@lo:400:kworker.0 00000100:00080000:2.0:1767319930.831234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319930.831486:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319930.831577:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941184 00000100:00100000:2.0:1767319930.831583:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941184 00000100:00100000:2.0:1767319930.831624:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 880 00000100:00100000:2.0:1767319930.831631:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982941184:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319930.831689:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982941184:12345-0@lo:400:kworker.0 Request processed in 58us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319930.831695:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 880 00000100:00100000:0.0:1767319930.831803:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982941184:0@lo:400:kworker.0 00000100:00100000:1.0:1767319930.831884:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319930.831989:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982940672 00000100:00100000:3.0:1767319930.831994:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982940672 00000100:00080000:2.0:1767319930.832079:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319930.832115:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1410 00000100:00100000:3.0:1767319930.832136:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982940672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319930.832184:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982940672:12345-0@lo:400:kworker.0 Request processed in 62us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319930.832190:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1410 00010000:00080000:1.0:1767319930.832247:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469f800 x1853163982940672/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319946 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319930.832257:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982940672:0@lo:400:kworker.0 00010000:00010000:1.0:1767319931.012562:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319931.788418:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319931.788520:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652608 00000100:00100000:2.0:1767319931.788533:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652608 00000100:00100000:2.0:1767319931.788616:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1862 00000100:00100000:2.0:1767319931.788657:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915652608:12345-192.168.204.7@tcp:17:kworker.0 00000100:00100000:2.0:1767319931.788763:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915652608:12345-192.168.204.7@tcp:17:kworker.0 Request processed in 138us (356us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319931.788770:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1862 00000100:00080000:3.0:1767319931.789756:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319932.036586:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319932.817525:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319932.817579:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319932.817718:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652864 00000100:00100000:0.0:1767319932.817728:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652864 00000100:00100000:0.0:1767319932.817783:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915652736 00000100:00100000:0.0:1767319932.817787:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915652736 00000100:00100000:0.0:1767319932.817821:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1411 00000100:00100000:0.0:1767319932.817832:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915652864:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319932.817910:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915652864:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (400us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319932.817918:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1411 00000100:00100000:0.0:1767319932.817947:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1863 00000100:00100000:0.0:1767319932.817952:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915652736:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319932.817972:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319932.817999:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915652736:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (419us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319932.818006:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1863 00010000:00010000:2.0:1767319933.061162:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319934.085139:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319935.108521:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319935.428687:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982941440:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319935.428785:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319935.429189:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941440 00000100:00100000:1.0:1767319935.429199:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941440 00000100:00100000:1.0:1767319935.429306:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952 00000100:00100000:1.0:1767319935.429316:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982941440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319935.429349:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319935.429395:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982941440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (610us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319935.429401:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952 00000100:00080000:3.0:1767319935.429590:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767319935.429631:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982941440:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319935.432645:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319935.432680:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982941568:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319935.432751:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319935.432924:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941568 00000100:00100000:2.0:1767319935.432942:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941568 00000100:00100000:2.0:1767319935.433015:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953 00000100:00100000:2.0:1767319935.433025:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982941568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319935.433057:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319935.433117:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982941568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319935.433124:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953 00000100:00080000:0.0:1767319935.433442:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319935.433496:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982941568:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767319935.940497:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319935.940509:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319935.940589:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319935.940593:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319935.940597:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319935.940606:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5179 00000100:00080000:3.0:1767319935.940612:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319935.940619:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319935.940656:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246300 x1853163982941824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319935.940722:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319935.940726:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319935.940729:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319935.940733:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5179 00000100:00080000:3.0:1767319935.940736:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319935.940741:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319935.940752:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358000 x1853163982942080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319935.940762:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319935.940765:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319935.940767:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319935.940771:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5179 00000100:00080000:3.0:1767319935.940774:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319935.940778:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319935.940789:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b800 x1853163982942208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319935.940810:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319935.940814:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319935.940816:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319935.940820:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5179 00000100:00080000:3.0:1767319935.940822:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319935.940825:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319935.940834:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358e00 x1853163982942592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319935.940877:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982941952:0@lo:400:kworker.0 00000100:00100000:3.0:1767319935.940939:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319935.940999:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319935.941006:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982942336:0@lo:400:kworker.0 00000100:00100000:3.0:1767319935.941032:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319935.941066:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942336 00000100:00100000:3.0:1767319935.941073:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942336 00000100:00100000:3.0:1767319935.941116:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 881 00000100:00100000:3.0:1767319935.941124:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982942336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.941314:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982941696:0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.941350:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319935.941392:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941952 00000100:00100000:1.0:1767319935.941393:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319935.941398:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941952 00000100:00100000:1.0:1767319935.941403:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982942464:0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.941430:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319935.941442:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941696 00000100:00100000:0.0:1767319935.941446:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941696 00000100:00100000:1.0:1767319935.941464:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942464 00000100:00100000:1.0:1767319935.941470:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942464 00000100:00100000:0.0:1767319935.941477:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1412 00000100:00100000:0.0:1767319935.941487:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982941952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.941503:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 882 00000100:00100000:1.0:1767319935.941512:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982942464:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319935.941563:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982941952:12345-0@lo:400:kworker.0 Request processed in 64us (609us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319935.941572:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1412 00000100:00100000:1.0:1767319935.941588:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982942464:12345-0@lo:400:kworker.0 Request processed in 75us (155us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319935.941595:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1413 00000100:00100000:1.0:1767319935.941600:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 882 00000100:00100000:0.0:1767319935.941605:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982941696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.941644:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982942464:0@lo:400:kworker.0 00000100:00080000:1.0:1767319935.941687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319935.941781:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982941824:0@lo:38:kworker.0 00000100:00100000:0.0:1767319935.945294:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982941696:12345-0@lo:400:kworker.0 Request processed in 3665us (3916us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319935.945315:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1413 00010000:00080000:0.0:1767319935.945395:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7100 x1853163982941696/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319951 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319935.946384:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982942336:12345-0@lo:400:kworker.0 Request processed in 5257us (5347us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319935.947262:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 881 00000100:00100000:0.0:1767319935.947352:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982941696:0@lo:400:kworker.0 00000100:00100000:2.0:1767319935.951342:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767319935.951343:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358380 x1853163982941952/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319951 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319935.951362:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982941952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319935.951398:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982942336:0@lo:400:kworker.0 00000100:00100000:2.0:1767319935.951420:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982942080:0@lo:38:kworker.0 00000100:00100000:2.0:1767319935.951452:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319935.951482:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982942208:0@lo:38:kworker.0 00000100:00100000:0.0:1767319935.951504:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982941824 00000100:00100000:2.0:1767319935.951511:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319935.951514:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982941824 00000100:00100000:2.0:1767319935.951545:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982942592:0@lo:38:kworker.0 00000100:00100000:2.0:1767319935.951580:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319935.951621:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942080 00000100:00100000:2.0:1767319935.951625:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942080 00000100:00100000:2.0:1767319935.951790:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942592 00000100:00100000:2.0:1767319935.951794:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942592 00000100:00100000:0.0:1767319935.951871:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864 00000100:00100000:0.0:1767319935.951880:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982942080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319935.951903:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319935.951952:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982942080:12345-0@lo:38:kworker.0 Request processed in 72us (500us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319935.951959:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864 00000100:00100000:0.0:1767319935.951976:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1865 00000100:00100000:0.0:1767319935.951979:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982941824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319935.951988:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319935.952012:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982941824:12345-0@lo:38:kworker.0 Request processed in 30us (678us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319935.952019:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1865 00000100:00100000:0.0:1767319935.952030:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866 00000100:00100000:0.0:1767319935.952033:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982942592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319935.952041:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319935.952063:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982942592:12345-0@lo:38:kworker.0 Request processed in 28us (482us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319935.952068:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866 00000100:00100000:1.0:1767319935.953260:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942208 00000100:00100000:1.0:1767319935.953299:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942208 00000100:00100000:1.0:1767319935.953339:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 491 00000100:00100000:1.0:1767319935.953346:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982942208:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319935.953362:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319935.953417:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982942208:12345-0@lo:38:kworker.0 Request processed in 70us (1906us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319935.953424:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 491 00000100:00080000:3.0:1767319935.959441:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319935.959454:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319935.959459:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982941824:0@lo:38:kworker.0 00000100:00080000:3.0:1767319935.959475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319935.959479:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319935.959482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982942080:0@lo:38:kworker.0 00000100:00080000:3.0:1767319935.959493:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319935.959497:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319935.959500:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982942208:0@lo:38:kworker.0 00000100:00080000:3.0:1767319935.959511:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319935.959514:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319935.959517:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982942592:0@lo:38:kworker.0 00010000:00010000:2.0:1767319936.132513:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319936.917068:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319936.931099:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653248 00000100:00100000:3.0:1767319936.931115:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653248 00000100:00100000:3.0:1767319936.931211:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1414 00000100:00100000:3.0:1767319936.931222:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915653248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319936.931350:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915653248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (14292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319936.931363:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1414 00000400:00000100:3.0:1767319936.931425:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915652992 offset 0 length 520: 4 00000100:00100000:3.0:1767319936.931521:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319936.931557:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653120 00000100:00100000:3.0:1767319936.931562:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653120 00000100:00100000:2.0:1767319936.931622:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1867 00000100:00100000:2.0:1767319936.931628:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915653120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319936.931655:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319936.931692:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915653120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 62us (171us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319936.931699:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1867 00000100:00080000:1.0:1767319936.935651:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319937.156520:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319938.181016:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319939.205203:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319940.036419:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319915/real 1767319915] req@ffff93460469dc00 x1853163982936448/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319940 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319940.036458:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460469dc00 x1853163982936448/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319940 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319940.036490:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.036518:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319940.036522:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982936448:0@lo:250:kworker.0 00000100:00080000:0.0:1767319940.036730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319940.036736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767319940.036740:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319940.036745:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5164 00000100:00080000:0.0:1767319940.036749:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319940.036765:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319940.036797:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c700 x1853163982942720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319940.036846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319940.036850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319940.036852:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319940.036856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5184 00000100:00080000:0.0:1767319940.036858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319940.036862:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319940.036880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f100 x1853163982942976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319940.036906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319940.036910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319940.036912:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319940.036915:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5184 00000100:00080000:0.0:1767319940.036918:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319940.036922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319940.036929:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319940.036933:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982943232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319940.036940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767319940.036941:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982942848:0@lo:400:kworker.0 00000100:00080000:0.0:1767319940.036943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319940.036945:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319940.036947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5184 00000100:00080000:0.0:1767319940.036949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319940.036953:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319940.036964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469dc00 x1853163982943360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319940.036985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319940.036988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319940.036989:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319940.036992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5184 00000100:00080000:0.0:1767319940.036994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319940.036997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319940.037006:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163982943744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319940.037008:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319940.037031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982943488:0@lo:400:kworker.0 00000100:00100000:0.0:1767319940.037068:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319940.037094:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319940.037096:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982943616:0@lo:400:kworker.0 00000100:00100000:0.0:1767319940.037098:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982943104:0@lo:400:kworker.0 00000100:00100000:0.0:1767319940.037119:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319940.037121:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319940.037151:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943488 00000100:00100000:2.0:1767319940.037151:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942848 00000100:00100000:0.0:1767319940.037160:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943488 00000100:00100000:2.0:1767319940.037160:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942848 00000100:00100000:2.0:1767319940.037212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943104 00000100:00100000:3.0:1767319940.037212:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982942720:0@lo:250:kworker.0 00000100:00100000:2.0:1767319940.037215:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943104 00000400:00000100:3.0:1767319940.037336:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982942720 offset 0 length 520: 4 00000100:00100000:3.0:1767319940.037357:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982942976:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.037379:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.037396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982943232:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.037416:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.037433:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982943360:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.037454:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.037470:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982943744:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.037503:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.037525:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982942976 00000100:00100000:3.0:1767319940.037530:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982942976 00000100:00100000:3.0:1767319940.037559:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943232 00000100:00100000:3.0:1767319940.037562:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943232 00000100:00100000:3.0:1767319940.037569:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943744 00000100:00100000:3.0:1767319940.037571:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943744 00000100:00100000:3.0:1767319940.037581:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1415 00000100:00100000:3.0:1767319940.037589:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982942848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319940.037664:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982942848:12345-0@lo:400:kworker.0 Request processed in 75us (656us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319940.037672:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1415 00000100:00100000:3.0:1767319940.037691:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1868 00000100:00100000:3.0:1767319940.037696:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982942976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.037716:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.037748:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982942976:12345-0@lo:38:kworker.0 Request processed in 51us (368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.037754:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1868 00000100:00100000:3.0:1767319940.037772:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1869 00000100:00100000:3.0:1767319940.037776:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982943232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.037785:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.037809:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982943232:12345-0@lo:38:kworker.0 Request processed in 31us (391us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.037814:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1869 00000100:00100000:3.0:1767319940.037826:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870 00000100:00100000:3.0:1767319940.037830:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982943744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.037838:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.037860:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982943744:12345-0@lo:38:kworker.0 Request processed in 28us (357us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.037865:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870 00000100:00100000:3.0:1767319940.037882:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943360 00000100:00100000:3.0:1767319940.037890:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943360 00000100:00100000:3.0:1767319940.037910:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 492 00000100:00100000:3.0:1767319940.037914:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982943360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319940.037923:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319940.037948:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982943360:12345-0@lo:38:kworker.0 Request processed in 33us (493us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319940.037954:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 492 00010000:00080000:3.0:1767319940.037987:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d500 x1853163982942848/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319956 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319940.037999:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982942848:0@lo:400:kworker.0 00000100:00080000:3.0:1767319940.038023:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.038033:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.038036:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982942976:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.038044:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.038047:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.038050:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982943232:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.038057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.038060:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.038062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982943360:0@lo:38:kworker.0 00000100:00080000:3.0:1767319940.038069:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319940.038072:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319940.038075:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982943744:0@lo:38:kworker.0 00000100:00100000:3.0:1767319940.038120:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943616 00000100:00100000:3.0:1767319940.038124:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943616 00000100:00100000:0.0:1767319940.038206:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 883 00000100:00100000:0.0:1767319940.038214:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982943488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319940.038252:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982943488:12345-0@lo:400:kworker.0 Request processed in 39us (1182us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319940.038258:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 883 00000100:00100000:0.0:1767319940.038301:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982943488:0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.038408:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1416 00000100:00100000:2.0:1767319940.038414:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982943104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319940.038450:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982943104:12345-0@lo:400:kworker.0 Request processed in 37us (1330us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319940.038456:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1416 00010000:00080000:2.0:1767319940.038492:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ea00 x1853163982943104/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319956 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319940.038502:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982943104:0@lo:400:kworker.0 00000100:00080000:1.0:1767319940.041814:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319940.041828:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319940.041833:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319940.042090:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 884 00000100:00100000:3.0:1767319940.042107:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982943616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319940.042220:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982943616:12345-0@lo:400:kworker.0 Request processed in 120us (5096us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319940.042228:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 884 00000100:00100000:2.0:1767319940.042314:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982943616:0@lo:400:kworker.0 00010000:00010000:1.0:1767319940.228717:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319940.548781:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982943872:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319940.548854:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319940.549081:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319940.549100:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982944000:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319940.549130:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319940.549267:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982943872 00000100:00100000:1.0:1767319940.549290:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982943872 00000100:00100000:1.0:1767319940.549340:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944000 00000100:00100000:1.0:1767319940.549343:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944000 00000100:00100000:1.0:1767319940.549358:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 954 00000100:00100000:1.0:1767319940.549365:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982943872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319940.549392:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319940.549446:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982943872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (592us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319940.549454:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 954 00000100:00100000:1.0:1767319940.549482:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 955 00000100:00100000:1.0:1767319940.549488:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982944000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319940.549503:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319940.549530:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982944000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 43us (400us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319940.549536:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 955 00000100:00100000:1.0:1767319940.549575:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982943872:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319940.549807:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982944000:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319941.253324:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319942.031732:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319942.031797:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319942.032003:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653376 00000100:00100000:2.0:1767319942.032016:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653376 00000100:00100000:2.0:1767319942.032069:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653504 00000100:00100000:2.0:1767319942.032072:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653504 00000100:00100000:2.0:1767319942.032133:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1417 00000100:00100000:2.0:1767319942.032143:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915653504:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319942.032221:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915653504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (422us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319942.032228:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1417 00000100:00100000:2.0:1767319942.032246:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1871 00000100:00100000:2.0:1767319942.032250:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915653376:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319942.032307:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319942.032334:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915653376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (613us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319942.032340:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1871 00000100:00080000:1.0:1767319942.032529:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319942.276858:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319943.300684:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319944.328057:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319945.156485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319945.156495:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767319945.156602:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982944128:0@lo:400:kworker.0 00000100:00100000:1.0:1767319945.156674:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319945.156734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319945.156738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319945.156742:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319945.156747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5188 00000100:00080000:3.0:1767319945.156750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319945.156756:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319945.157105:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944128 00000100:00100000:2.0:1767319945.157113:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944128 00000100:00100000:2.0:1767319945.157170:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1418 00000100:00100000:2.0:1767319945.157180:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982944128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319945.157251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358700 x1853163982944256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319945.157317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319945.157322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319945.157324:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319945.157328:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5188 00000100:00080000:3.0:1767319945.157330:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319945.157333:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319945.157346:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b100 x1853163982944512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319945.157354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319945.157357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319945.157359:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319945.157362:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5188 00000100:00080000:3.0:1767319945.157364:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319945.157367:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319945.157377:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359f80 x1853163982944640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319945.157466:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319945.157491:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982944896:0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.157525:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.157560:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944896 00000100:00100000:0.0:1767319945.157564:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944896 00000100:00100000:0.0:1767319945.157597:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 885 00000100:00100000:0.0:1767319945.157605:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982944896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.157640:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982944896:12345-0@lo:400:kworker.0 Request processed in 37us (114us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319945.157647:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 885 00000100:00100000:0.0:1767319945.157670:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982944256:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.157692:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.157707:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982944512:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.157726:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.157742:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982944640:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.157764:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.157810:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944256 00000100:00100000:0.0:1767319945.157814:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944256 00000100:00100000:0.0:1767319945.157832:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944512 00000100:00100000:0.0:1767319945.157835:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944512 00000100:00100000:0.0:1767319945.157850:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872 00000100:00100000:0.0:1767319945.157855:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982944256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319945.157875:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319945.157906:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982944256:12345-0@lo:38:kworker.0 Request processed in 50us (212us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319945.157912:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872 00000100:00100000:0.0:1767319945.157924:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1873 00000100:00100000:0.0:1767319945.157928:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982944512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319945.157937:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319945.157973:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982944512:12345-0@lo:38:kworker.0 Request processed in 43us (246us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319945.157979:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1873 00000100:00100000:0.0:1767319945.157997:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944640 00000100:00100000:0.0:1767319945.158000:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944640 00000100:00100000:0.0:1767319945.158018:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 493 00000100:00100000:0.0:1767319945.158022:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982944640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319945.158031:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319945.158053:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982944640:12345-0@lo:38:kworker.0 Request processed in 30us (288us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319945.158058:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 493 00000100:00080000:0.0:1767319945.158077:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767319945.158103:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982944896:0@lo:400:kworker.0 00000100:00080000:0.0:1767319945.158131:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319945.158136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319945.158140:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982944256:0@lo:38:kworker.0 00000100:00080000:0.0:1767319945.158148:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319945.158151:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319945.158154:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982944512:0@lo:38:kworker.0 00000100:00080000:0.0:1767319945.158161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319945.158163:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319945.158166:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982944640:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.158215:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982944768:0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.158263:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.158303:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319945.158309:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982944384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.158330:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.158355:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944768 00000100:00100000:0.0:1767319945.158359:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944768 00000100:00100000:0.0:1767319945.158398:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 886 00000100:00100000:0.0:1767319945.158405:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982944768:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.158439:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982944768:12345-0@lo:400:kworker.0 Request processed in 34us (174us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319945.158445:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 886 00000100:00100000:0.0:1767319945.158460:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982944384 00000100:00100000:0.0:1767319945.158463:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982944384 00000100:00100000:0.0:1767319945.158487:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1419 00000100:00100000:0.0:1767319945.158492:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982944384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.158519:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982944384:12345-0@lo:400:kworker.0 Request processed in 27us (188us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319945.158525:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1419 00000100:00080000:0.0:1767319945.158539:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767319945.158542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:0.0:1767319945.158561:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435aa00 x1853163982944384/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319961 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319945.158572:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982944384:0@lo:400:kworker.0 00000100:00100000:0.0:1767319945.158582:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982944768:0@lo:400:kworker.0 00000100:00100000:2.0:1767319945.161233:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982944128:12345-0@lo:400:kworker.0 Request processed in 4044us (4547us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319945.161250:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1418 00000100:00080000:3.0:1767319945.161509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319945.161516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319945.161520:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319945.161525:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5188 00000100:00080000:3.0:1767319945.161529:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319945.161535:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319945.161580:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435a680 x1853163982945024/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319945.161896:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982945024:0@lo:38:kworker.0 00000100:00100000:0.0:1767319945.162034:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319945.162296:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945024 00000100:00100000:2.0:1767319945.162305:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945024 00000100:00100000:2.0:1767319945.162345:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1874 00000100:00100000:2.0:1767319945.162353:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982945024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319945.162373:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319945.162439:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982945024:12345-0@lo:38:kworker.0 Request processed in 84us (404us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319945.162445:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1874 00000100:00080000:3.0:1767319945.162704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319945.162711:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319945.162715:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982945024:0@lo:38:kworker.0 00010000:00080000:1.0:1767319945.166511:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604359500 x1853163982944128/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319961 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319945.166533:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982944128:0@lo:400:kworker.0 00010000:00010000:2.0:1767319945.349138:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319945.668614:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319945.668655:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982945152:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319945.668693:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982945280:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319945.668767:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319945.668818:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945152 00000100:00100000:1.0:1767319945.668830:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945152 00000100:00100000:1.0:1767319945.668896:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956 00000100:00100000:1.0:1767319945.668906:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982945152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319945.668938:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319945.675565:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982945152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 6651us (6791us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319945.675579:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956 00000100:00100000:2.0:1767319945.678378:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319945.678906:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982945152:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319945.679140:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945280 00000100:00100000:1.0:1767319945.679156:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945280 00000100:00100000:1.0:1767319945.679331:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957 00000100:00100000:1.0:1767319945.679342:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982945280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319945.679385:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319945.679475:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982945280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 134us (1117us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319945.679481:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957 00000100:00100000:2.0:1767319945.679580:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982945280:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319946.373071:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319947.147915:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319947.148041:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319947.163440:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653760 00000100:00100000:0.0:1767319947.163465:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653760 00000100:00080000:1.0:1767319947.163571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319947.163607:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653632 00000100:00100000:0.0:1767319947.163612:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653632 00000100:00100000:0.0:1767319947.163642:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1420 00000100:00100000:0.0:1767319947.163655:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915653760:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319947.163657:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1875 00000100:00100000:1.0:1767319947.163663:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915653632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319947.163691:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319947.163770:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915653632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 106us (15728us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319947.163778:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1875 00000100:00100000:0.0:1767319947.169666:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915653760:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 5992us (21745us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319947.170641:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1420 00010000:00010000:2.0:1767319947.396758:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319948.420711:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319949.445995:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319950.277145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.277161:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319950.277256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.277261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.277266:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767319950.277317:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319950.277361:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982945408:0@lo:400:kworker.0 00000100:00080000:1.0:1767319950.277382:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.277389:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.277400:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319950.277438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1180 x1853163982945536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319950.277476:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319950.277483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.277488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.277490:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.277495:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.277499:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.277506:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319950.277522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1880 x1853163982945792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319950.277532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.277537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.277539:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.277543:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.277546:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.277551:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319950.277552:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319950.277566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2300 x1853163982945920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319950.277569:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982945664:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.277624:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319950.277653:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982946048:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.277679:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319950.277710:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945408 00000100:00100000:3.0:1767319950.277719:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945408 00000100:00100000:3.0:1767319950.277772:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945664 00000100:00100000:3.0:1767319950.277775:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945664 00000100:00100000:3.0:1767319950.277801:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1421 00000100:00100000:3.0:1767319950.277809:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982945408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.277865:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982945408:12345-0@lo:400:kworker.0 Request processed in 57us (393us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.277873:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1421 00000100:00100000:3.0:1767319950.277891:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1422 00000100:00100000:3.0:1767319950.277897:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982945664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.277928:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982945664:12345-0@lo:400:kworker.0 Request processed in 31us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319950.277933:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1422 00010000:00080000:3.0:1767319950.277968:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a2d80 x1853163982945664/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319966 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319950.277988:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982945664:0@lo:400:kworker.0 00010000:00080000:3.0:1767319950.278024:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3b80 x1853163982945408/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319966 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319950.278031:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946048 00000100:00100000:3.0:1767319950.278036:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982945408:0@lo:400:kworker.0 00000100:00100000:0.0:1767319950.278042:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946048 00000100:00080000:3.0:1767319950.278103:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767319950.278112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319950.278114:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 887 00000100:00100000:0.0:1767319950.278126:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982946048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319950.278228:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982946048:12345-0@lo:400:kworker.0 Request processed in 104us (548us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319950.278237:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 887 00000100:00100000:3.0:1767319950.278306:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982946048:0@lo:400:kworker.0 00000100:00100000:2.0:1767319950.279026:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982945536:0@lo:38:kworker.0 00000100:00080000:1.0:1767319950.279458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319950.279464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319950.279467:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319950.279472:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5193 00000100:00080000:1.0:1767319950.279476:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319950.279483:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319950.279505:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0700 x1853163982946304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319950.279918:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.279966:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982945792:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.279993:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.280016:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982945920:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.280043:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.280092:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982946304:0@lo:38:kworker.0 00000100:00100000:2.0:1767319950.280113:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.280150:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945536 00000100:00100000:2.0:1767319950.280159:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945536 00000100:00100000:2.0:1767319950.280190:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945792 00000100:00100000:2.0:1767319950.280193:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945792 00000100:00100000:2.0:1767319950.280218:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946304 00000100:00100000:2.0:1767319950.280221:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946304 00000100:00100000:2.0:1767319950.280237:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876 00000100:00100000:2.0:1767319950.280246:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982945536:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319950.280337:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1877 00000100:00100000:3.0:1767319950.280348:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982945792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319950.280374:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319950.280462:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982945792:12345-0@lo:38:kworker.0 Request processed in 113us (466us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319950.280469:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1877 00000100:00100000:3.0:1767319950.280492:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878 00000100:00100000:3.0:1767319950.280496:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982946304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319950.280505:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319950.280541:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982946304:12345-0@lo:38:kworker.0 Request processed in 32us (415us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319950.280547:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878 00000100:00100000:3.0:1767319950.280572:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982946176:0@lo:400:kworker.0 00000100:00100000:3.0:1767319950.280609:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767319950.280687:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319950.280692:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319950.280696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982946304:0@lo:38:kworker.0 00000100:00080000:3.0:1767319950.280706:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319950.280709:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319950.280712:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982945792:0@lo:38:kworker.0 00000100:00100000:0.0:1767319950.280807:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982945920 00000100:00100000:0.0:1767319950.280814:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982945920 00000100:00100000:0.0:1767319950.280857:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 494 00000100:00100000:0.0:1767319950.280864:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982945920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319950.280876:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319950.280933:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982945920:12345-0@lo:38:kworker.0 Request processed in 69us (889us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319950.280940:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 494 00000100:00100000:0.0:1767319950.280982:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946176 00000100:00100000:0.0:1767319950.280986:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946176 00000100:00100000:0.0:1767319950.281036:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 888 00000100:00100000:0.0:1767319950.281043:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982946176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319950.281095:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982946176:12345-0@lo:400:kworker.0 Request processed in 52us (485us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319950.281101:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 888 00000100:00080000:0.0:1767319950.281124:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767319950.281153:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319950.281158:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319950.281163:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982945920:0@lo:38:kworker.0 00010000:00020000:2.0:1767319950.281420:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319950.281468:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982945536:12345-0@lo:38:kworker.0 Request processed in 1221us (1550us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319950.281475:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876 00000100:00080000:0.0:1767319950.284906:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767319950.292731:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982946176:0@lo:400:kworker.0 00000100:00080000:0.0:1767319950.302947:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319950.302962:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982945536:0@lo:38:kworker.0 00010000:00010000:2.0:1767319950.468733:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319950.788962:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319950.788993:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982946432:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319950.789407:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319950.789864:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946432 00000100:00100000:2.0:1767319950.789876:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946432 00000100:00100000:2.0:1767319950.789991:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 958 00000100:00100000:2.0:1767319950.790004:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982946432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319950.790057:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319950.790116:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982946432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 115us (709us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319950.790124:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 958 00000100:00100000:3.0:1767319950.790195:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982946432:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319950.790248:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319950.790262:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982946560:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767319950.790322:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319950.794202:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946560 00000100:00100000:0.0:1767319950.794380:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946560 00000100:00100000:0.0:1767319950.794548:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959 00000100:00100000:0.0:1767319950.794562:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982946560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319950.794884:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319950.795040:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982946560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 481us (4716us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319950.795050:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959 00000100:00100000:1.0:1767319950.797090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982946560:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319951.496894:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319952.270121:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319952.270234:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319952.270704:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915653888 00000100:00100000:3.0:1767319952.270718:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915653888 00000100:00100000:3.0:1767319952.270772:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654016 00000100:00100000:3.0:1767319952.270777:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654016 00000100:00100000:3.0:1767319952.270842:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1423 00000100:00100000:3.0:1767319952.270855:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915654016:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319952.270955:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915654016:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (719us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319952.270964:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1423 00000100:00100000:3.0:1767319952.270988:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1879 00000100:00100000:3.0:1767319952.270994:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915653888:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319952.271029:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319952.271059:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915653888:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (954us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319952.271066:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1879 00000100:00080000:1.0:1767319952.273308:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319952.516556:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319953.540641:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319954.564526:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319955.396465:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319955.396478:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319955.396538:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319955.396543:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319955.396547:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319955.396555:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5198 00000100:00080000:3.0:1767319955.396559:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319955.396566:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319955.396598:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358700 x1853163982946816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319955.396639:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319955.396642:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319955.396644:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319955.396648:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5198 00000100:00080000:3.0:1767319955.396650:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319955.396654:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319955.396667:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ea00 x1853163982947072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319955.396676:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319955.396679:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319955.396681:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319955.396685:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5198 00000100:00080000:3.0:1767319955.396687:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319955.396691:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319955.396703:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f800 x1853163982947200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319955.396731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982946688:0@lo:400:kworker.0 00000100:00080000:3.0:1767319955.396735:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319955.396738:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319955.396739:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319955.396742:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5198 00000100:00080000:3.0:1767319955.396744:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319955.396747:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319955.396759:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f100 x1853163982947584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319955.396779:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982946944:0@lo:400:kworker.0 00000100:00100000:1.0:1767319955.396796:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.396847:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319955.396856:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767319955.396861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982947456:0@lo:400:kworker.0 00000100:00100000:3.0:1767319955.396865:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767319955.396869:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982947328:0@lo:400:kworker.0 00000100:00100000:0.0:1767319955.397045:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946688 00000100:00100000:0.0:1767319955.397053:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946688 00000100:00100000:0.0:1767319955.397106:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946944 00000100:00100000:0.0:1767319955.397109:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946944 00000100:00100000:0.0:1767319955.397135:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1424 00000100:00100000:0.0:1767319955.397144:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982946688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319955.398053:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1425 00000100:00100000:2.0:1767319955.398065:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982946944:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319955.398456:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.398522:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982946816:0@lo:38:kworker.0 00000100:00100000:3.0:1767319955.398575:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.398603:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982947072:0@lo:38:kworker.0 00000100:00100000:3.0:1767319955.398624:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.398638:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982947200:0@lo:38:kworker.0 00000100:00100000:3.0:1767319955.398661:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.398681:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982947584:0@lo:38:kworker.0 00000100:00100000:3.0:1767319955.398697:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.398719:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982946816 00000100:00100000:3.0:1767319955.398738:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982946816 00000100:00100000:3.0:1767319955.398757:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947072 00000100:00100000:3.0:1767319955.398759:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947072 00000100:00100000:3.0:1767319955.398766:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947584 00000100:00100000:3.0:1767319955.398768:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947584 00000100:00100000:3.0:1767319955.398778:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1880 00000100:00100000:3.0:1767319955.398783:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982946816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319955.398801:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319955.398990:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982946816:12345-0@lo:38:kworker.0 Request processed in 205us (413us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319955.398997:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1880 00000100:00100000:3.0:1767319955.399015:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1881 00000100:00100000:3.0:1767319955.399018:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982947072:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319955.399029:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319955.399054:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982947072:12345-0@lo:38:kworker.0 Request processed in 34us (429us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319955.399059:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1881 00000100:00100000:3.0:1767319955.399070:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882 00000100:00100000:3.0:1767319955.399073:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982947584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319955.399082:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319955.399103:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982947584:12345-0@lo:38:kworker.0 Request processed in 29us (405us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319955.399108:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882 00000100:00100000:3.0:1767319955.399127:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947200 00000100:00100000:3.0:1767319955.399130:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947200 00000100:00100000:3.0:1767319955.399149:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 495 00000100:00100000:3.0:1767319955.399153:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982947200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319955.399163:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319955.399187:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982947200:12345-0@lo:38:kworker.0 Request processed in 32us (525us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319955.399227:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 495 00000100:00080000:3.0:1767319955.399266:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319955.399292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319955.399296:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982946816:0@lo:38:kworker.0 00000100:00080000:3.0:1767319955.399307:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319955.399311:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319955.399314:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982947072:0@lo:38:kworker.0 00000100:00080000:3.0:1767319955.399321:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319955.399325:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319955.399327:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982947200:0@lo:38:kworker.0 00000100:00080000:3.0:1767319955.399335:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319955.399338:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319955.399341:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982947584:0@lo:38:kworker.0 00000100:00100000:3.0:1767319955.399398:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947328 00000100:00100000:3.0:1767319955.399402:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947328 00000100:00100000:3.0:1767319955.399434:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 889 00000100:00100000:3.0:1767319955.399441:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982947328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319955.399478:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982947328:12345-0@lo:400:kworker.0 Request processed in 38us (1022us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.399485:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 889 00000100:00100000:3.0:1767319955.399510:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982947328:0@lo:400:kworker.0 00000100:00100000:1.0:1767319955.399732:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319955.399773:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319955.399777:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767319955.402795:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947456 00000100:00100000:3.0:1767319955.402811:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947456 00000100:00100000:3.0:1767319955.404202:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 890 00000100:00100000:3.0:1767319955.404222:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982947456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319955.404357:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982947456:12345-0@lo:400:kworker.0 Request processed in 142us (4625us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.404365:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 890 00000100:00080000:1.0:1767319955.404626:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319955.404676:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982947456:0@lo:400:kworker.0 00000100:00100000:0.0:1767319955.406218:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982946688:12345-0@lo:400:kworker.0 Request processed in 9067us (9413us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319955.406233:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1424 00000100:00100000:2.0:1767319955.414137:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982946944:12345-0@lo:400:kworker.0 Request processed in 16066us (17279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319955.414154:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1425 00010000:00080000:3.0:1767319955.414355:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358380 x1853163982946944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319971 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319955.414370:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982946944:0@lo:400:kworker.0 00010000:00080000:1.0:1767319955.418246:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b100 x1853163982946688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319971 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319955.418265:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982946688:0@lo:400:kworker.0 00010000:00010000:1.0:1767319955.595824:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319955.908595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982947712:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319955.908668:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.909391:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947712 00000100:00100000:3.0:1767319955.909401:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947712 00000100:00100000:2.0:1767319955.909582:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319955.909594:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982947840:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319955.909635:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319955.917548:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947840 00000100:00100000:3.0:1767319955.917563:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947840 00000100:00100000:3.0:1767319955.917616:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960 00000100:00100000:3.0:1767319955.917627:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982947712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319955.917665:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319955.917730:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982947712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (9061us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.917737:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960 00000100:00100000:3.0:1767319955.917769:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 961 00000100:00100000:3.0:1767319955.917777:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982947840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319955.917791:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319955.917823:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982947840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (8188us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319955.917829:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 961 00000100:00100000:2.0:1767319955.920755:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982947840:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319955.920806:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982947712:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319956.612810:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319957.392567:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319957.392674:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319957.394471:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654272 00000100:00100000:2.0:1767319957.394485:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654272 00000100:00080000:1.0:1767319957.394614:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319957.394648:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654144 00000100:00100000:2.0:1767319957.394653:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654144 00000100:00100000:0.0:1767319957.397949:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1426 00000100:00100000:0.0:1767319957.397975:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915654272:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319957.398061:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915654272:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 99us (5505us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319957.398068:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1426 00000100:00100000:0.0:1767319957.398090:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1883 00000100:00100000:0.0:1767319957.398095:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915654144:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319957.398114:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319957.398136:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915654144:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (5462us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319957.398142:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1883 00010000:00010000:2.0:1767319957.638449:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319958.660609:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319959.684581:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319960.518423:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319960.518434:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319960.518502:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319960.518506:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319960.518511:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319960.518521:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5203 00000100:00080000:2.0:1767319960.518526:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319960.518533:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319960.518555:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2a00 x1853163982948096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319960.518595:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319960.518598:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319960.518600:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319960.518603:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5203 00000100:00080000:2.0:1767319960.518606:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319960.518610:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319960.518619:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0700 x1853163982948352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319960.518628:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319960.518630:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319960.518632:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319960.518636:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5203 00000100:00080000:2.0:1767319960.518638:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319960.518642:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319960.518650:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3100 x1853163982948480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767319960.518670:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319960.518673:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319960.518675:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319960.518678:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5203 00000100:00080000:2.0:1767319960.518680:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319960.518683:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319960.518690:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163982948864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319960.518720:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982948608:0@lo:400:kworker.0 00000100:00100000:2.0:1767319960.518780:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319960.518839:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767319960.518845:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982948224:0@lo:400:kworker.0 00000100:00100000:2.0:1767319960.518866:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.518951:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982947968:0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.518979:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.519012:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319960.519017:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982948736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.519050:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319960.519080:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948608 00000100:00100000:0.0:1767319960.519089:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948608 00000100:00100000:0.0:1767319960.519182:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948736 00000100:00100000:0.0:1767319960.519185:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948736 00000100:00100000:0.0:1767319960.519205:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 891 00000100:00100000:0.0:1767319960.519214:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982948608:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.519258:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982948608:12345-0@lo:400:kworker.0 Request processed in 46us (478us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319960.519265:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 891 00000100:00100000:0.0:1767319960.519300:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 892 00000100:00100000:0.0:1767319960.519306:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982948736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.519333:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982948736:12345-0@lo:400:kworker.0 Request processed in 28us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319960.519339:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 892 00000100:00100000:0.0:1767319960.519377:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982948736:0@lo:400:kworker.0 00000100:00100000:0.0:1767319960.519400:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982948608:0@lo:400:kworker.0 00000100:00080000:1.0:1767319960.519471:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319960.519476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767319960.519599:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948224 00000100:00100000:2.0:1767319960.519603:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948224 00000100:00100000:2.0:1767319960.519680:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982947968 00000100:00100000:2.0:1767319960.519684:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982947968 00000100:00100000:3.0:1767319960.519737:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1427 00000100:00100000:3.0:1767319960.519746:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982948224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319960.519785:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982948224:12345-0@lo:400:kworker.0 Request processed in 41us (918us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319960.519790:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1427 00000100:00100000:3.0:1767319960.519813:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982948096:0@lo:38:kworker.0 00000100:00100000:3.0:1767319960.519837:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319960.519857:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982948352:0@lo:38:kworker.0 00000100:00100000:3.0:1767319960.519873:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319960.519888:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982948480:0@lo:38:kworker.0 00000100:00100000:3.0:1767319960.519905:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319960.519922:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982948864:0@lo:38:kworker.0 00000100:00100000:3.0:1767319960.519940:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319960.519962:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948096 00000100:00100000:3.0:1767319960.519965:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948096 00000100:00100000:3.0:1767319960.519976:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948352 00000100:00100000:3.0:1767319960.519978:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948352 00000100:00100000:3.0:1767319960.519984:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948864 00000100:00100000:3.0:1767319960.519986:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948864 00000100:00100000:3.0:1767319960.519994:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884 00000100:00100000:3.0:1767319960.519998:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319960.520015:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319960.520043:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948096:12345-0@lo:38:kworker.0 Request processed in 43us (204us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319960.520048:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884 00000100:00100000:3.0:1767319960.520059:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1885 00000100:00100000:3.0:1767319960.520063:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948352:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319960.520071:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319960.520089:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948352:12345-0@lo:38:kworker.0 Request processed in 24us (215us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319960.520174:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1885 00000100:00100000:3.0:1767319960.520185:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1886 00000100:00100000:3.0:1767319960.520189:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319960.520198:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319960.520217:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982948864:12345-0@lo:38:kworker.0 Request processed in 28us (277us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319960.520223:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1886 00000100:00100000:3.0:1767319960.520238:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948480 00000100:00100000:3.0:1767319960.520242:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948480 00000100:00100000:3.0:1767319960.520258:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 496 00000100:00100000:3.0:1767319960.520263:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982948480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319960.520292:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319960.520314:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982948480:12345-0@lo:38:kworker.0 Request processed in 51us (408us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319960.520320:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 496 00010000:00080000:3.0:1767319960.520355:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2d80 x1853163982948224/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319976 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319960.520365:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982948224:0@lo:400:kworker.0 00000100:00080000:3.0:1767319960.520385:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319960.520389:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319960.520392:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982948096:0@lo:38:kworker.0 00000100:00080000:3.0:1767319960.520399:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319960.520402:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319960.520404:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982948352:0@lo:38:kworker.0 00000100:00080000:3.0:1767319960.520410:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319960.520413:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319960.520416:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982948480:0@lo:38:kworker.0 00000100:00080000:3.0:1767319960.520422:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319960.520425:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319960.520427:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982948864:0@lo:38:kworker.0 00000100:00080000:1.0:1767319960.525644:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767319960.526638:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1428 00000100:00100000:2.0:1767319960.526656:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982947968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319960.526774:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982947968:12345-0@lo:400:kworker.0 Request processed in 125us (7793us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319960.526782:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1428 00010000:00080000:3.0:1767319960.526876:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1c00 x1853163982947968/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319976 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319960.526894:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982947968:0@lo:400:kworker.0 00010000:00010000:2.0:1767319960.713433:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319961.028528:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319961.028551:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982948992:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319961.028620:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319961.029230:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982949120:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319961.029260:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319961.029367:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982948992 00000100:00100000:2.0:1767319961.029375:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982948992 00000100:00100000:2.0:1767319961.029423:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949120 00000100:00100000:2.0:1767319961.029426:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949120 00000100:00100000:2.0:1767319961.029494:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 962 00000100:00100000:2.0:1767319961.029504:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982948992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319961.029536:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319961.029581:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982948992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (960us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319961.029588:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 962 00000100:00100000:2.0:1767319961.029604:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963 00000100:00100000:2.0:1767319961.029609:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982949120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319961.029624:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319961.029646:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982949120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 37us (385us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319961.029651:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963 00000100:00100000:0.0:1767319961.029821:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982949120:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319961.029850:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982948992:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319961.732497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319962.515990:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319962.516595:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319962.517111:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654528 00000100:00100000:3.0:1767319962.517123:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654528 00000100:00100000:3.0:1767319962.517235:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654400 00000100:00100000:3.0:1767319962.517239:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654400 00000100:00100000:3.0:1767319962.517263:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1429 00000100:00100000:3.0:1767319962.517297:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915654528:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767319962.517376:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915654528:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (2379us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319962.517383:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1429 00000100:00100000:3.0:1767319962.517415:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1887 00000100:00100000:3.0:1767319962.517420:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915654400:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319962.517440:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319962.517466:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915654400:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (873us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319962.517484:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1887 00000100:00080000:1.0:1767319962.518872:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767319962.526861:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915654656 offset 0 length 520: 4 00010000:00010000:2.0:1767319962.756566:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319963.780570:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319964.804520:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319965.636510:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319940/real 1767319940] req@ffff93460469c700 x1853163982942720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319965 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319965.636568:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460469c700 x1853163982942720/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319965 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767319965.636583:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.636612:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767319965.636616:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982942720:0@lo:250:kworker.0 00000100:00080000:1.0:1767319965.636832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319965.636837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319965.636840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319965.636845:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5188 00000100:00080000:1.0:1767319965.636848:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319965.636855:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319965.636888:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0e00 x1853163982949248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319965.636936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319965.636939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319965.636941:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319965.636944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5209 00000100:00080000:1.0:1767319965.636946:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319965.636950:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319965.636962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1180 x1853163982949504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319965.636982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319965.636985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319965.636987:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319965.636990:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5209 00000100:00080000:1.0:1767319965.636992:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319965.636996:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319965.637018:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2300 x1853163982949760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319965.637026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319965.637030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319965.637032:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319965.637034:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5209 00000100:00080000:1.0:1767319965.637036:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319965.637040:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319965.637050:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1880 x1853163982949888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319965.637068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319965.637072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319965.637073:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319965.637077:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5209 00000100:00080000:1.0:1767319965.637079:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319965.637082:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319965.637091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3b80 x1853163982950272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319965.637158:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982949376:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637221:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.637306:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319965.637313:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982950144:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637338:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.637368:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950144 00000100:00100000:3.0:1767319965.637375:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950144 00000100:00100000:3.0:1767319965.637426:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 893 00000100:00100000:3.0:1767319965.637435:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982950144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637489:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982950144:12345-0@lo:400:kworker.0 Request processed in 54us (149us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319965.637498:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 893 00000100:00100000:3.0:1767319965.637527:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982950016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637567:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.637587:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319965.637593:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982949632:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637615:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.637638:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950016 00000100:00100000:3.0:1767319965.637642:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950016 00000100:00100000:3.0:1767319965.637667:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 894 00000100:00100000:3.0:1767319965.637673:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982950016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637719:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982950016:12345-0@lo:400:kworker.0 Request processed in 46us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319965.637729:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 894 00000100:00100000:3.0:1767319965.637760:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982950016:0@lo:400:kworker.0 00000100:00100000:3.0:1767319965.637783:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982950144:0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.638441:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949376 00000100:00100000:1.0:1767319965.638447:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949376 00000100:00100000:1.0:1767319965.638488:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949632 00000100:00100000:1.0:1767319965.638491:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949632 00000100:00100000:1.0:1767319965.638515:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1430 00000100:00100000:1.0:1767319965.638522:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982949376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.638586:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982949376:12345-0@lo:400:kworker.0 Request processed in 64us (1364us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319965.638593:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1430 00000100:00100000:1.0:1767319965.638607:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1431 00000100:00100000:1.0:1767319965.638612:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982949632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319965.638641:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982949632:12345-0@lo:400:kworker.0 Request processed in 29us (1025us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319965.638647:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1431 00000100:00080000:1.0:1767319965.638663:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767319965.638668:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319965.638671:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:1.0:1767319965.638708:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3800 x1853163982949376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319981 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319965.638720:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982949376:0@lo:400:kworker.0 00010000:00080000:3.0:1767319965.638968:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1500 x1853163982949632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319981 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319965.638978:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982949632:0@lo:400:kworker.0 00000100:00100000:0.0:1767319965.639473:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982949248:0@lo:250:kworker.0 00000400:00000100:0.0:1767319965.639509:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982949248 offset 0 length 520: 4 00000100:00100000:0.0:1767319965.639555:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982949504:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.639583:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.639612:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982949760:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.639633:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319965.639655:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982949888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.639676:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319965.639884:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949504 00000100:00100000:3.0:1767319965.639891:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949504 00000100:00100000:3.0:1767319965.639918:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949760 00000100:00100000:3.0:1767319965.639921:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949760 00000100:00100000:3.0:1767319965.639938:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888 00000100:00100000:3.0:1767319965.639944:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982949504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319965.639963:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319965.640010:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982949504:12345-0@lo:38:kworker.0 Request processed in 54us (415us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319965.640017:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888 00000100:00100000:3.0:1767319965.640031:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1889 00000100:00100000:3.0:1767319965.640035:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982949760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319965.640045:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319965.640070:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982949760:12345-0@lo:38:kworker.0 Request processed in 33us (436us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319965.640076:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1889 00000100:00100000:3.0:1767319965.640129:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982949888 00000100:00100000:3.0:1767319965.640133:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982949888 00000100:00100000:3.0:1767319965.640151:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 497 00000100:00100000:3.0:1767319965.640156:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982949888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319965.640165:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319965.645193:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982949888:12345-0@lo:38:kworker.0 Request processed in 5024us (5503us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319965.645210:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 497 00000100:00080000:0.0:1767319965.645360:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.645366:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319965.645370:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982949888:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.645381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982950272:0@lo:38:kworker.0 00000100:00100000:0.0:1767319965.645449:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767319965.645508:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.645512:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319965.645515:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982949504:0@lo:38:kworker.0 00000100:00080000:0.0:1767319965.645526:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.645529:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319965.645531:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982949760:0@lo:38:kworker.0 00000100:00100000:1.0:1767319965.645948:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950272 00000100:00100000:1.0:1767319965.645956:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950272 00000100:00100000:1.0:1767319965.646012:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890 00000100:00100000:1.0:1767319965.646019:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982950272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319965.646039:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319965.646105:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982950272:12345-0@lo:38:kworker.0 Request processed in 85us (656us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319965.646112:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890 00000100:00080000:0.0:1767319965.646224:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319965.646229:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319965.646232:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982950272:0@lo:38:kworker.0 00010000:00010000:2.0:1767319965.828681:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319966.148954:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319966.148984:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982950400:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319966.149069:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319966.149339:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982950528:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319966.149383:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319966.149723:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950400 00000100:00100000:1.0:1767319966.149732:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950400 00000100:00100000:1.0:1767319966.149783:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950528 00000100:00100000:1.0:1767319966.149787:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950528 00000100:00100000:1.0:1767319966.149804:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 964 00000100:00100000:1.0:1767319966.149813:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982950400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767319966.149844:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319966.149900:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982950400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (830us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319966.149907:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 964 00000100:00100000:1.0:1767319966.149925:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965 00000100:00100000:1.0:1767319966.149932:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346325d0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982950528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319966.149948:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319966.149979:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346325d0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982950528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (596us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319966.149985:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965 00000100:00100000:0.0:1767319966.150261:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982950400:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319966.150766:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982950528:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319966.852648:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319967.627936:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319967.628014:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319967.628226:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654784 00000100:00100000:1.0:1767319967.628241:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654784 00000100:00100000:1.0:1767319967.628364:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915654912 00000100:00100000:1.0:1767319967.628368:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915654912 00000100:00100000:1.0:1767319967.628433:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1432 00000100:00100000:1.0:1767319967.628446:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915654912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319967.628552:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915654912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 108us (536us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319967.628562:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1432 00000100:00100000:1.0:1767319967.628588:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1891 00000100:00100000:1.0:1767319967.628593:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915654784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319967.628626:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319967.628671:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915654784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 75us (751us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319967.628678:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1891 00000100:00080000:2.0:1767319967.629080:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767319967.876799:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319968.900905:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319969.924613:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767319970.756663:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319970.756679:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767319970.756949:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319970.756954:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319970.756959:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319970.756971:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5214 00000100:00080000:2.0:1767319970.756978:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319970.756987:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319970.757019:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0e00 x1853163982950784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319970.757082:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982950784:0@lo:38:kworker.0 00000100:00100000:1.0:1767319970.757181:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319970.757322:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982950656:0@lo:400:kworker.0 00000100:00100000:3.0:1767319970.757360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319970.757413:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950784 00000100:00100000:3.0:1767319970.757423:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950784 00000100:00100000:3.0:1767319970.757468:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950656 00000100:00100000:3.0:1767319970.757470:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950656 00000100:00100000:3.0:1767319970.757526:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1433 00000100:00100000:3.0:1767319970.757538:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982950656:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319970.757597:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982950656:12345-0@lo:400:kworker.0 Request processed in 62us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319970.757606:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1433 00000100:00100000:3.0:1767319970.757628:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1892 00000100:00100000:3.0:1767319970.757633:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982950784:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319970.757657:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319970.757791:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767319970.758760:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319970.758766:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319970.758769:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319970.758777:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5214 00000100:00080000:2.0:1767319970.758782:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319970.758788:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319970.758813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0000 x1853163982951040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767319970.758900:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982950912:0@lo:400:kworker.0 00000100:00100000:0.0:1767319970.758978:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319970.759047:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982950912 00000100:00100000:0.0:1767319970.759054:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982950912 00000100:00100000:0.0:1767319970.759108:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1434 00000100:00100000:0.0:1767319970.759116:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982950912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319970.759169:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982950912:12345-0@lo:400:kworker.0 Request processed in 54us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319970.759178:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1434 00000100:00080000:0.0:1767319970.759204:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767319970.759244:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0700 x1853163982950912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319986 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319970.759260:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982950912:0@lo:400:kworker.0 00010000:00080000:0.0:1767319970.759340:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af1f80 x1853163982950656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319986 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319970.759351:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982950656:0@lo:400:kworker.0 00000100:00100000:1.0:1767319970.761047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982951040:0@lo:38:kworker.0 00000100:00100000:1.0:1767319970.761152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319970.761352:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951040 00000100:00100000:0.0:1767319970.761369:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951040 00000100:00100000:0.0:1767319970.761434:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893 00000100:00100000:0.0:1767319970.761447:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982951040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319970.761487:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319970.761565:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982951040:12345-0@lo:38:kworker.0 Request processed in 118us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319970.761573:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893 00000100:00100000:3.0:1767319970.762491:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982950784:12345-0@lo:38:kworker.0 Request processed in 4848us (5301us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319970.762521:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1892 00000100:00080000:1.0:1767319970.766378:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319970.766399:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319970.766405:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982950784:0@lo:38:kworker.0 00000100:00080000:1.0:1767319970.766464:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319970.766469:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319970.766473:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982951040:0@lo:38:kworker.0 00000100:00080000:2.0:1767319970.766700:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319970.766854:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319970.766858:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319970.766863:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5214 00000100:00080000:2.0:1767319970.766868:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319970.766877:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319970.766969:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1c00 x1853163982951168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319970.767035:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982951168:0@lo:38:kworker.0 00000100:00100000:1.0:1767319970.767126:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767319970.767266:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767319970.767293:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767319970.767296:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767319970.767301:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5214 00000100:00080000:2.0:1767319970.767304:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767319970.767309:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319970.767322:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163982951552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319970.767354:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319970.767367:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982951296:0@lo:400:kworker.0 00000100:00100000:2.0:1767319970.767403:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319970.767459:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951296 00000100:00100000:2.0:1767319970.767468:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951296 00000100:00100000:2.0:1767319970.767566:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 895 00000100:00100000:2.0:1767319970.767578:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982951296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319970.767638:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982951296:12345-0@lo:400:kworker.0 Request processed in 61us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319970.767648:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 895 00000100:00080000:2.0:1767319970.767676:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319970.767854:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767319970.767867:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982951424:0@lo:400:kworker.0 00000100:00100000:0.0:1767319970.767913:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319970.768062:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982951552:0@lo:38:kworker.0 00000100:00100000:1.0:1767319970.768093:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319970.768179:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951552 00000100:00100000:0.0:1767319970.768184:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951552 00000100:00100000:0.0:1767319970.768225:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894 00000100:00100000:0.0:1767319970.768232:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982951552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319970.768261:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319970.768331:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982951552:12345-0@lo:38:kworker.0 Request processed in 98us (237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319970.768338:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894 00000100:00100000:0.0:1767319970.768366:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951424 00000100:00100000:0.0:1767319970.768370:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951424 00000100:00100000:0.0:1767319970.768401:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896 00000100:00100000:0.0:1767319970.768409:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982951424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319970.768447:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982951424:12345-0@lo:400:kworker.0 Request processed in 40us (542us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319970.768453:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896 00000100:00100000:0.0:1767319970.768482:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982951424:0@lo:400:kworker.0 00000100:00080000:0.0:1767319970.768513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767319970.768842:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982951296:0@lo:400:kworker.0 00000100:00080000:1.0:1767319970.769070:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319970.769076:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319970.769080:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982951552:0@lo:38:kworker.0 00000100:00100000:3.0:1767319970.778000:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951168 00000100:00100000:3.0:1767319970.778032:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951168 00000100:00100000:3.0:1767319970.778135:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 498 00000100:00100000:3.0:1767319970.778147:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982951168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319970.778183:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319970.778365:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982951168:12345-0@lo:38:kworker.0 Request processed in 218us (11237us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319970.778375:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 498 00000100:00080000:1.0:1767319970.778776:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319970.778785:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319970.778790:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982951168:0@lo:38:kworker.0 00010000:00010000:1.0:1767319970.950965:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767319971.268703:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982951680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319971.268808:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319971.268934:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951680 00000100:00100000:2.0:1767319971.268952:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951680 00000100:00100000:2.0:1767319971.269026:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 966 00000100:00100000:2.0:1767319971.269038:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982951680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767319971.269082:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319971.269243:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982951680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 207us (435us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319971.269251:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 966 00000100:00100000:3.0:1767319971.269840:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319971.269860:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982951808:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319971.269928:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319971.270014:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951808 00000100:00100000:1.0:1767319971.270024:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951808 00000100:00100000:1.0:1767319971.270082:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 967 00000100:00100000:1.0:1767319971.270093:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982951808:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767319971.270125:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767319971.270173:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982951808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319971.270181:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 967 00000100:00100000:3.0:1767319971.270303:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982951808:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319971.270550:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982951680:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767319971.972792:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319972.749651:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319972.749769:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319972.749943:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655168 00000100:00100000:0.0:1767319972.749954:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655168 00000100:00100000:0.0:1767319972.750031:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655040 00000100:00100000:0.0:1767319972.750036:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655040 00000100:00080000:2.0:1767319972.750067:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767319972.750068:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1435 00000100:00100000:0.0:1767319972.750147:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915655168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319972.750238:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915655168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 154us (597us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319972.751825:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1895 00000100:00100000:3.0:1767319972.751837:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915655040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319972.751861:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319972.751950:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915655040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 112us (2181us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319972.751959:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1895 00000100:00100000:0.0:1767319972.753878:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1435 00010000:00010000:2.0:1767319972.997155:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319974.020533:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319975.045542:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319975.879321:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319975.879333:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767319975.879441:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982951936:0@lo:400:kworker.0 00000100:00100000:0.0:1767319975.879522:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319975.879600:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982951936 00000100:00100000:2.0:1767319975.879608:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982951936 00000100:00100000:2.0:1767319975.879680:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1436 00000100:00100000:2.0:1767319975.879690:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982951936:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767319975.879765:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319975.880327:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319975.880334:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319975.880338:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319975.880344:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5219 00000100:00080000:3.0:1767319975.880349:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319975.880354:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319975.880380:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9180 x1853163982952064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319975.880414:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319975.880418:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319975.880420:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319975.880423:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5219 00000100:00080000:3.0:1767319975.880426:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319975.880429:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319975.880440:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9880 x1853163982952320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319975.880450:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319975.880452:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319975.880454:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319975.880457:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5219 00000100:00080000:3.0:1767319975.880459:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319975.880463:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319975.880475:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e300 x1853163982952448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319975.880579:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982952704:0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.880624:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982952192:0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.880690:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880707:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319975.880712:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982952576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.880732:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880756:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982952064:0@lo:38:kworker.0 00000100:00100000:1.0:1767319975.880779:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880795:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982952320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319975.880828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880847:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982952448:0@lo:38:kworker.0 00000100:00100000:1.0:1767319975.880871:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319975.880911:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952192 00000100:00100000:1.0:1767319975.880916:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952192 00000100:00100000:1.0:1767319975.880956:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952064 00000100:00100000:1.0:1767319975.880959:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952064 00000100:00100000:1.0:1767319975.880967:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952320 00000100:00100000:1.0:1767319975.880969:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952320 00000100:00100000:1.0:1767319975.880979:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1437 00000100:00100000:1.0:1767319975.880986:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982952192:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.881024:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982952192:12345-0@lo:400:kworker.0 Request processed in 39us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319975.881032:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1437 00000100:00100000:1.0:1767319975.881048:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1767319975.881053:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982952064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319975.881072:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319975.881101:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982952064:12345-0@lo:38:kworker.0 Request processed in 46us (322us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319975.881107:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1767319975.881118:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897 00000100:00100000:1.0:1767319975.881122:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982952320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319975.881131:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319975.881157:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982952320:12345-0@lo:38:kworker.0 Request processed in 33us (328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319975.881163:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897 00000100:00080000:1.0:1767319975.881184:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319975.881208:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ead80 x1853163982952192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319991 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319975.881221:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982952192:0@lo:400:kworker.0 00000100:00080000:1.0:1767319975.881252:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319975.881257:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319975.881260:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982952064:0@lo:38:kworker.0 00000100:00080000:1.0:1767319975.881294:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319975.881299:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319975.881302:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982952320:0@lo:38:kworker.0 00000100:00100000:1.0:1767319975.881323:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952704 00000100:00100000:1.0:1767319975.881326:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952704 00000100:00100000:1.0:1767319975.881357:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952576 00000100:00100000:1.0:1767319975.881360:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952576 00000100:00100000:1.0:1767319975.881379:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 897 00000100:00100000:1.0:1767319975.881385:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982952704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.881418:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982952704:12345-0@lo:400:kworker.0 Request processed in 33us (794us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319975.881424:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 897 00000100:00100000:1.0:1767319975.881435:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 898 00000100:00100000:1.0:1767319975.881440:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982952576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.881468:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982952576:12345-0@lo:400:kworker.0 Request processed in 28us (735us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319975.881474:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 898 00000100:00080000:1.0:1767319975.881492:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319975.881495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767319975.881510:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982952704:0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.881528:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982952576:0@lo:400:kworker.0 00000100:00100000:1.0:1767319975.881592:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952448 00000100:00100000:1.0:1767319975.881596:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952448 00000100:00100000:1.0:1767319975.881617:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 499 00000100:00100000:1.0:1767319975.881621:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982952448:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319975.881632:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319975.881660:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982952448:12345-0@lo:38:kworker.0 Request processed in 38us (788us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319975.881667:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 499 00000100:00080000:1.0:1767319975.881685:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319975.881689:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319975.881692:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982952448:0@lo:38:kworker.0 00000100:00100000:2.0:1767319975.883760:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982951936:12345-0@lo:400:kworker.0 Request processed in 4061us (4227us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319975.883775:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1436 00010000:00080000:0.0:1767319975.886362:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ebb80 x1853163982951936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319991 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767319975.886381:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982951936:0@lo:400:kworker.0 00000100:00080000:3.0:1767319975.887736:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319975.887745:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319975.887750:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319975.887757:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5219 00000100:00080000:3.0:1767319975.887761:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319975.887768:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319975.887806:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982952832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319975.888388:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982952832:0@lo:38:kworker.0 00000100:00100000:1.0:1767319975.888480:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319975.888661:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952832 00000100:00100000:0.0:1767319975.888670:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952832 00000100:00100000:0.0:1767319975.888729:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1898 00000100:00100000:0.0:1767319975.888737:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982952832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767319975.888760:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319975.899687:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982952832:12345-0@lo:38:kworker.0 Request processed in 10935us (11194us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319975.899704:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1898 00000100:00080000:1.0:1767319975.901989:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767319975.902004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767319975.902009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982952832:0@lo:38:kworker.0 00010000:00010000:2.0:1767319976.070392:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319976.388782:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319976.388795:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982953088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319976.388813:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982952960:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319976.390569:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319976.390621:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953088 00000100:00100000:0.0:1767319976.390632:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953088 00000100:00100000:0.0:1767319976.390687:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968 00000100:00100000:0.0:1767319976.390698:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982953088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319976.390734:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319976.390798:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982953088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (236us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319976.390816:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968 00000100:00100000:3.0:1767319976.390924:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982953088:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319976.391261:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319976.395353:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982952960 00000100:00100000:0.0:1767319976.395371:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982952960 00000100:00100000:0.0:1767319976.395442:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969 00000100:00100000:0.0:1767319976.395453:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982952960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319976.395869:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319976.395967:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982952960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 516us (4705us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319976.395975:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969 00000100:00100000:2.0:1767319976.402793:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982952960:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767319977.093404:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319977.869627:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319977.869739:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319977.869851:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655424 00000100:00100000:0.0:1767319977.869869:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655424 00000100:00100000:0.0:1767319977.869954:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655296 00000100:00100000:0.0:1767319977.869958:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655296 00000100:00100000:0.0:1767319977.869981:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1438 00000100:00100000:0.0:1767319977.869991:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915655424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319977.870080:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915655424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (466us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319977.870089:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1438 00000100:00100000:0.0:1767319977.870110:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1899 00000100:00100000:0.0:1767319977.870114:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915655296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767319977.870142:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319977.870170:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915655296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (430us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319977.870176:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1899 00000100:00080000:1.0:1767319977.870263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319978.116594:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319979.140518:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319980.168297:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767319980.996502:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319980.996515:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767319980.996588:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319980.996593:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319980.996597:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319980.996608:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5224 00000100:00080000:3.0:1767319980.996613:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319980.996621:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319980.996645:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f800 x1853163982953344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319980.996695:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319980.996698:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319980.996700:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319980.996704:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5224 00000100:00080000:3.0:1767319980.996707:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319980.996711:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319980.996733:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ce00 x1853163982953600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319980.996743:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319980.996746:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319980.996748:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319980.996751:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5224 00000100:00080000:3.0:1767319980.996754:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319980.996758:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319980.996773:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469e300 x1853163982953728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767319980.996795:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767319980.996798:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767319980.996800:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767319980.996803:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5224 00000100:00080000:3.0:1767319980.996806:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767319980.996810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767319980.996821:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359880 x1853163982954112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319980.996933:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982953984:0@lo:400:kworker.0 00000100:00100000:1.0:1767319980.997013:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319980.997095:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319980.997102:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982953216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319980.997129:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997223:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982953344:0@lo:38:kworker.0 00000100:00100000:2.0:1767319980.997253:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997309:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982953600:0@lo:38:kworker.0 00000100:00100000:2.0:1767319980.997329:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982953728:0@lo:38:kworker.0 00000100:00100000:2.0:1767319980.997372:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997390:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982954112:0@lo:38:kworker.0 00000100:00100000:2.0:1767319980.997411:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997438:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982953472:0@lo:400:kworker.0 00000100:00100000:2.0:1767319980.997460:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997475:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767319980.997481:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982953856:0@lo:400:kworker.0 00000100:00100000:2.0:1767319980.997504:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319980.997531:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953216 00000100:00100000:2.0:1767319980.997538:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953216 00000100:00100000:2.0:1767319980.997580:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953344 00000100:00100000:2.0:1767319980.997583:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953344 00000100:00100000:2.0:1767319980.997592:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953600 00000100:00100000:2.0:1767319980.997595:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953600 00000100:00100000:2.0:1767319980.997601:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954112 00000100:00100000:2.0:1767319980.997603:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954112 00000100:00100000:2.0:1767319980.997610:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953472 00000100:00100000:2.0:1767319980.997612:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953472 00000100:00100000:2.0:1767319980.997628:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1439 00000100:00100000:2.0:1767319980.997636:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982953216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319980.997686:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982953216:12345-0@lo:400:kworker.0 Request processed in 50us (556us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319980.997693:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1439 00000100:00100000:2.0:1767319980.997708:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1440 00000100:00100000:2.0:1767319980.997724:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982953472:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319980.997755:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982953472:12345-0@lo:400:kworker.0 Request processed in 41us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319980.997760:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1440 00000100:00100000:2.0:1767319980.997772:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900 00000100:00100000:2.0:1767319980.997776:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982953344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319980.997791:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319980.997820:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982953344:12345-0@lo:38:kworker.0 Request processed in 43us (566us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319980.997826:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900 00000100:00100000:2.0:1767319980.997837:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1901 00000100:00100000:2.0:1767319980.997840:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982953600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319980.997849:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319980.997869:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982953600:12345-0@lo:38:kworker.0 Request processed in 27us (539us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319980.997874:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1901 00000100:00100000:2.0:1767319980.997884:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902 00000100:00100000:2.0:1767319980.997887:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982954112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767319980.997896:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319980.997916:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982954112:12345-0@lo:38:kworker.0 Request processed in 27us (505us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319980.997922:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902 00010000:00080000:2.0:1767319980.997961:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ea00 x1853163982953472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319996 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767319980.997974:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982953472:0@lo:400:kworker.0 00000100:00080000:2.0:1767319980.998003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319980.998008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319980.998011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982953344:0@lo:38:kworker.0 00000100:00080000:2.0:1767319980.998020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319980.998023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319980.998026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982953600:0@lo:38:kworker.0 00000100:00080000:2.0:1767319980.998032:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767319980.998035:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767319980.998038:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982954112:0@lo:38:kworker.0 00000100:00100000:1.0:1767319980.998200:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953728 00000100:00100000:1.0:1767319980.998205:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953728 00000100:00100000:1.0:1767319980.998237:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 500 00000100:00100000:1.0:1767319980.998242:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982953728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319980.998254:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319980.998322:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982953728:12345-0@lo:38:kworker.0 Request processed in 78us (948us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319980.998328:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 500 00000100:00100000:1.0:1767319980.998354:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953984 00000100:00100000:1.0:1767319980.998357:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953984 00000100:00100000:1.0:1767319980.998383:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982953856 00000100:00100000:1.0:1767319980.998387:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982953856 00000100:00100000:1.0:1767319980.998407:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 899 00000100:00100000:1.0:1767319980.998413:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982953984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319980.998444:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982953984:12345-0@lo:400:kworker.0 Request processed in 32us (1431us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319980.998450:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899 00000100:00100000:1.0:1767319980.998460:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900 00000100:00100000:1.0:1767319980.998466:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982953856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319980.998497:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982953856:12345-0@lo:400:kworker.0 Request processed in 31us (992us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319980.998503:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900 00000100:00080000:1.0:1767319980.998517:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319980.998521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319980.998524:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319980.998550:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982953856:0@lo:400:kworker.0 00010000:00080000:1.0:1767319980.998575:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469f100 x1853163982953216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767319996 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319980.998585:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982953216:0@lo:400:kworker.0 00000100:00100000:1.0:1767319980.998595:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982953984:0@lo:400:kworker.0 00000100:00080000:0.0:1767319980.998661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767319980.998666:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767319980.998668:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982953728:0@lo:38:kworker.0 00010000:00010000:2.0:1767319981.188704:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319981.509139:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767319981.509177:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982954240:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767319981.509264:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319981.509404:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954240 00000100:00100000:3.0:1767319981.509412:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954240 00000100:00100000:3.0:1767319981.509468:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 970 00000100:00100000:3.0:1767319981.509478:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982954240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319981.509510:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319981.509562:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982954240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (297us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319981.509570:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 970 00000100:00100000:1.0:1767319981.509683:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982954240:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319981.510090:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767319981.510101:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982954368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319981.510133:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319981.510153:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954368 00000100:00100000:0.0:1767319981.510617:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954368 00000100:00100000:0.0:1767319981.510716:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971 00000100:00100000:0.0:1767319981.510725:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982954368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767319981.510748:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319981.510816:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982954368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (681us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319981.510822:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971 00000100:00100000:1.0:1767319981.510894:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982954368:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767319982.213530:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319982.988363:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319982.988481:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655680 00000100:00100000:1.0:1767319982.988495:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655680 00000100:00100000:1.0:1767319982.988587:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1441 00000100:00100000:1.0:1767319982.988605:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915655680:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319982.988697:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915655680:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319982.988706:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1441 00000100:00080000:2.0:1767319982.988927:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319982.990752:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319982.990850:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655552 00000100:00100000:3.0:1767319982.990861:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655552 00000100:00100000:3.0:1767319982.990909:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1903 00000100:00100000:3.0:1767319982.990917:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915655552:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767319982.990945:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319982.991028:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915655552:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 108us (286us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319982.991036:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1903 00010000:00010000:1.0:1767319983.236746:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319984.260537:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319985.284625:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767319986.116931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319986.116943:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767319986.117009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319986.117015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319986.117019:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319986.117027:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5229 00000100:00080000:0.0:1767319986.117031:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319986.117038:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319986.117064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909500 x1853163982954624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319986.117106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319986.117110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319986.117112:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319986.117116:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5229 00000100:00080000:0.0:1767319986.117118:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319986.117122:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319986.117133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909f80 x1853163982954880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767319986.117142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767319986.117146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319986.117148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319986.117151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5229 00000100:00080000:0.0:1767319986.117154:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319986.117159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319986.117171:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190a680 x1853163982955008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319986.117178:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767319986.117201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767319986.117202:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982954496:0@lo:400:kworker.0 00000100:00080000:0.0:1767319986.117208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767319986.117210:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767319986.117215:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5229 00000100:00080000:0.0:1767319986.117218:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767319986.117224:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319986.117237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163982955392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319986.117241:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982954624:0@lo:38:kworker.0 00000100:00100000:0.0:1767319986.117299:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982955136:0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.117325:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319986.117342:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319986.117359:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954496 00000100:00100000:2.0:1767319986.117368:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954496 00000100:00100000:3.0:1767319986.117396:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982955264:0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.117431:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319986.117432:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982954752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319986.117436:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1442 00000100:00100000:2.0:1767319986.117450:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982954496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.117474:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955136 00000100:00100000:1.0:1767319986.117476:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319986.117481:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955136 00000100:00100000:2.0:1767319986.117515:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982954496:12345-0@lo:400:kworker.0 Request processed in 68us (190us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319986.117521:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955264 00000100:00100000:2.0:1767319986.117524:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1442 00000100:00100000:3.0:1767319986.117526:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955264 00000100:00100000:1.0:1767319986.117545:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 901 00000100:00100000:2.0:1767319986.117547:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954752 00000100:00100000:2.0:1767319986.117552:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954752 00000100:00100000:1.0:1767319986.117559:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982955136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319986.117571:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902 00000100:00100000:3.0:1767319986.117582:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604772450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982955264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767319986.117586:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1443 00000100:00100000:2.0:1767319986.117597:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982954752:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319986.117604:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982955136:12345-0@lo:400:kworker.0 Request processed in 49us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319986.117612:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 901 00000100:00100000:3.0:1767319986.117645:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604772450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982955264:12345-0@lo:400:kworker.0 Request processed in 65us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319986.117654:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902 00000100:00100000:2.0:1767319986.117658:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982954752:12345-0@lo:400:kworker.0 Request processed in 61us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319986.117666:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1443 00000100:00100000:2.0:1767319986.117693:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319986.117698:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767319986.117704:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767319986.117708:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767319986.117741:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190b100 x1853163982954752/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320002 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319986.117755:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982954752:0@lo:400:kworker.0 00000100:00100000:2.0:1767319986.117815:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982954880:0@lo:38:kworker.0 00000100:00100000:2.0:1767319986.117846:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319986.117889:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982955008:0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.117893:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982955264:0@lo:400:kworker.0 00010000:00080000:3.0:1767319986.117910:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190ad80 x1853163982954496/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320002 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319986.117922:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982954496:0@lo:400:kworker.0 00000100:00100000:2.0:1767319986.117923:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319986.117961:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982955392:0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.117964:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954624 00000100:00100000:3.0:1767319986.117969:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954624 00000100:00100000:3.0:1767319986.117986:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1904 00000100:00100000:3.0:1767319986.117992:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982954624:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767319986.118002:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767319986.118011:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319986.118034:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955008 00000100:00100000:2.0:1767319986.118040:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955008 00000100:00100000:3.0:1767319986.118050:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982954624:12345-0@lo:38:kworker.0 Request processed in 56us (356us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319986.118058:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1904 00000100:00100000:2.0:1767319986.118066:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 501 00000100:00100000:2.0:1767319986.118072:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982955008:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319986.118084:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982954880 00000100:00100000:3.0:1767319986.118089:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982954880 00000100:00100000:3.0:1767319986.118106:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955392 00000100:00100000:3.0:1767319986.118109:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955392 00000100:00100000:3.0:1767319986.118119:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1905 00000100:00100000:3.0:1767319986.118124:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982954880:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319986.118133:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319986.118157:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982954880:12345-0@lo:38:kworker.0 Request processed in 32us (311us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319986.118163:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1905 00000100:00100000:3.0:1767319986.118174:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906 00000100:00100000:3.0:1767319986.118177:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982955392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319986.118185:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319986.118205:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982955392:12345-0@lo:38:kworker.0 Request processed in 27us (205us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319986.118209:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906 00000100:00080000:3.0:1767319986.118236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.118241:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.118244:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982955392:0@lo:38:kworker.0 00000100:00080000:3.0:1767319986.118251:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.118254:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.118257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982954624:0@lo:38:kworker.0 00000100:00080000:3.0:1767319986.118264:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.118266:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.118290:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982954880:0@lo:38:kworker.0 00000100:00100000:0.0:1767319986.118565:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982955136:0@lo:400:kworker.0 00010000:00020000:2.0:1767319986.118887:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319986.118935:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982955008:12345-0@lo:38:kworker.0 Request processed in 861us (1014us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319986.118943:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 501 00000100:00080000:3.0:1767319986.118996:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319986.119001:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319986.119004:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982955008:0@lo:38:kworker.0 00010000:00010000:1.0:1767319986.308538:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767319986.628693:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767319986.628715:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982955648:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319986.628804:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319986.628887:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955648 00000100:00100000:2.0:1767319986.628895:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955648 00000100:00100000:2.0:1767319986.628943:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972 00000100:00100000:2.0:1767319986.628952:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982955648:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767319986.628982:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767319986.629027:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982955648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (223us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319986.629034:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972 00000100:00100000:2.0:1767319986.629077:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982955648:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319986.629379:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767319986.629392:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982955520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319986.629429:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319986.629550:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955520 00000100:00100000:3.0:1767319986.629555:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955520 00000100:00100000:3.0:1767319986.629606:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 973 00000100:00100000:3.0:1767319986.629622:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982955520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767319986.629641:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319986.629686:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982955520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319986.629693:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 973 00000100:00100000:3.0:1767319986.629726:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982955520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319987.084754:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767319987.084795:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915655808 offset 0 length 520: 4 00000100:00100000:0.0:1767319987.084818:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767319987.084820:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656064 00000100:00100000:1.0:1767319987.084831:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656064 00000100:00100000:1.0:1767319987.084965:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915655936 00000100:00100000:1.0:1767319987.084970:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915655936 00000100:00100000:1.0:1767319987.084992:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1444 00000100:00100000:2.0:1767319987.085000:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1907 00000100:00100000:1.0:1767319987.085002:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915656064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767319987.085006:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915655936:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767319987.085026:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767319987.085326:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915655936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 317us (505us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319987.085335:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1907 00000100:00080000:3.0:1767319987.094182:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319987.096135:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915656064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 11114us (11385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319987.098285:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1444 00010000:00010000:2.0:1767319987.332483:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319988.356744:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319989.381128:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319990.212688:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319965/real 1767319965] req@ffff9346333a0e00 x1853163982949248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319990 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319990.212728:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346333a0e00 x1853163982949248/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767319990 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767319990.212749:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.212777:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767319990.212781:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982949248:0@lo:250:kworker.0 00000100:00080000:1.0:1767319990.214364:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319990.214374:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767319990.214378:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319990.214384:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5214 00000100:00080000:1.0:1767319990.214389:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319990.214396:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319990.214431:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35880 x1853163982955776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319990.214503:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982955776:0@lo:250:kworker.0 00000100:00080000:1.0:1767319990.214509:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319990.214514:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319990.214516:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319990.214521:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5234 00000100:00080000:1.0:1767319990.214524:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319990.214528:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319990.214540:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982955904:0@lo:400:kworker.0 00000400:00000100:3.0:1767319990.214571:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982955776 offset 0 length 520: 4 00000100:00100000:1.0:1767319990.214574:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35500 x1853163982956032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319990.214595:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319990.214615:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319990.214620:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319990.214622:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319990.214626:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5234 00000100:00100000:3.0:1767319990.214630:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982956032:0@lo:38:kworker.0 00000100:00080000:1.0:1767319990.214631:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319990.214639:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319990.214656:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37800 x1853163982956288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319990.214658:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982956160:0@lo:400:kworker.0 00000100:00100000:3.0:1767319990.214674:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767319990.214678:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319990.214682:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319990.214684:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319990.214689:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5234 00000100:00080000:1.0:1767319990.214692:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319990.214696:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767319990.214696:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982955904 00000100:00100000:2.0:1767319990.214704:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982955904 00000100:00100000:3.0:1767319990.214704:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982956288:0@lo:38:kworker.0 00000100:00100000:1.0:1767319990.214714:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0000 x1853163982956416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767319990.214730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319990.214735:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956032 00000100:00100000:0.0:1767319990.214740:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956032 00000100:00100000:3.0:1767319990.214757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982956416:0@lo:38:kworker.0 00000100:00100000:2.0:1767319990.214762:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956288 00000100:00100000:2.0:1767319990.214765:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956288 00000100:00080000:1.0:1767319990.214768:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319990.214772:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767319990.214774:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1445 00000100:00080000:1.0:1767319990.214774:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319990.214779:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5234 00000100:00080000:1.0:1767319990.214781:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319990.214786:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767319990.214788:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982955904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319990.214791:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319990.214794:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908 00000100:00100000:1.0:1767319990.214803:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982956800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767319990.214805:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982956032:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767319990.214823:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982956544:0@lo:400:kworker.0 00010000:00020000:2.0:1767319990.214830:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319990.214851:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982955904:12345-0@lo:400:kworker.0 Request processed in 68us (256us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319990.214854:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1909 00000100:00100000:0.0:1767319990.214859:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1445 00000100:00100000:1.0:1767319990.214859:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982956288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767319990.214872:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319990.214873:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319990.214902:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982956672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.214916:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982956288:12345-0@lo:38:kworker.0 Request processed in 55us (187us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319990.214920:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956416 00000100:00100000:1.0:1767319990.214926:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1909 00000100:00100000:3.0:1767319990.214926:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956416 00000100:00100000:0.0:1767319990.214931:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319990.214956:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 502 00000100:00100000:1.0:1767319990.214959:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956544 00000100:00100000:3.0:1767319990.214964:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982956416:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767319990.214965:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956544 00010000:00020000:3.0:1767319990.214978:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767319990.214981:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767319990.214986:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319990.214994:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956672 00000100:00100000:1.0:1767319990.214998:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956672 00000100:00100000:0.0:1767319990.215003:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 903 00000100:00100000:0.0:1767319990.215012:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982956544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319990.215016:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982956416:12345-0@lo:38:kworker.0 Request processed in 50us (228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319990.215024:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 904 00000100:00100000:3.0:1767319990.215029:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 502 00000100:00100000:1.0:1767319990.215034:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982956672:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319990.215055:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982956544:12345-0@lo:400:kworker.0 Request processed in 45us (186us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319990.215061:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982956800:0@lo:38:kworker.0 00000100:00100000:0.0:1767319990.215066:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 903 00000100:00100000:1.0:1767319990.215069:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982956672:12345-0@lo:400:kworker.0 Request processed in 35us (138us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319990.215074:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 904 00000100:00080000:0.0:1767319990.215089:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767319990.215098:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319990.215118:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982956672:0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.215118:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982956544:0@lo:400:kworker.0 00000100:00080000:3.0:1767319990.215136:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767319990.215142:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956800 00000100:00100000:0.0:1767319990.215147:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956800 00000100:00100000:0.0:1767319990.215163:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1910 00000100:00100000:0.0:1767319990.215167:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982956800:12345-0@lo:38:kworker.0 00010000:00080000:1.0:1767319990.215185:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a37100 x1853163982955904/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320006 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00020000:0.0:1767319990.215191:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319990.215193:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982955904:0@lo:400:kworker.0 00000100:00100000:0.0:1767319990.215217:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982956800:12345-0@lo:38:kworker.0 Request processed in 48us (121us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319990.215217:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319990.215223:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1910 00000100:00100000:0.0:1767319990.215246:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956160 00000100:00100000:0.0:1767319990.215249:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956160 00000100:00080000:1.0:1767319990.215294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319990.215309:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1446 00000100:00100000:1.0:1767319990.215317:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982956160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319990.215350:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982956160:12345-0@lo:400:kworker.0 Request processed in 36us (133us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319990.215356:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1446 00010000:00080000:1.0:1767319990.215381:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34e00 x1853163982956160/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320006 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767319990.215389:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982956160:0@lo:400:kworker.0 00000100:00080000:3.0:1767319990.218520:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.218539:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982956416:0@lo:38:kworker.0 00000100:00080000:3.0:1767319990.219120:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.219127:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.219131:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982956288:0@lo:38:kworker.0 00000100:00080000:3.0:1767319990.219172:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.219176:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.219178:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982956800:0@lo:38:kworker.0 00000100:00100000:2.0:1767319990.242360:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982956032:12345-0@lo:38:kworker.0 Request processed in 27549us (27680us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767319990.242376:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908 00000100:00080000:3.0:1767319990.242819:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319990.242828:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319990.242832:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982956032:0@lo:38:kworker.0 00010000:00010000:2.0:1767319990.404510:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319991.428765:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767319991.748745:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319991.748884:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767319991.748918:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982956928:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319991.748993:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319991.749175:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982957056:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319991.749202:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982956928 00000100:00100000:1.0:1767319991.749210:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767319991.749215:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982956928 00000100:00100000:0.0:1767319991.749295:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957056 00000100:00100000:0.0:1767319991.749301:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957056 00000100:00100000:2.0:1767319991.749310:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 974 00000100:00100000:2.0:1767319991.749323:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982956928:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767319991.749330:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975 00000100:00100000:0.0:1767319991.749339:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982957056:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319991.749364:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:2.0:1767319991.749365:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319991.749417:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982957056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (207us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319991.749427:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975 00000100:00100000:2.0:1767319991.749429:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982956928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 108us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319991.749439:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 974 00000100:00100000:0.0:1767319991.749479:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982957056:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767319991.749484:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982956928:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767319992.203929:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767319992.204027:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319992.204549:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656192 00000100:00100000:0.0:1767319992.204561:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656192 00000100:00100000:0.0:1767319992.204696:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656320 00000100:00100000:0.0:1767319992.204700:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656320 00000100:00080000:2.0:1767319992.204772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767319992.204899:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1447 00000100:00100000:1.0:1767319992.204911:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915656320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767319992.204991:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915656320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (963us total) trans 0 rc 0/0 00000100:00100000:1.0:1767319992.204999:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1447 00000100:00100000:1.0:1767319992.205021:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1911 00000100:00100000:1.0:1767319992.205026:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915656192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767319992.205047:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767319992.205074:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915656192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (1155us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767319992.205080:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1911 00010000:00010000:1.0:1767319992.452522:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319993.476605:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319994.501298:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767319995.332494:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319995.332506:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767319995.332887:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319995.332892:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319995.332896:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319995.332904:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767319995.332909:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319995.332916:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319995.332942:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1180 x1853163982957312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319995.332981:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319995.332985:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319995.332987:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319995.332990:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767319995.332992:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319995.332996:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319995.333007:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37800 x1853163982957568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319995.333014:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319995.333017:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319995.333018:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319995.333022:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767319995.333024:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319995.333027:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319995.333037:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34e00 x1853163982957696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767319995.333059:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767319995.333062:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767319995.333064:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767319995.333068:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767319995.333070:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767319995.333072:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767319995.333085:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982958080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767319995.333135:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982957824:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.333213:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319995.333257:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767319995.333263:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982957440:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.333312:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767319995.333339:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957824 00000100:00100000:1.0:1767319995.333345:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957824 00000100:00080000:2.0:1767319995.333779:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767319995.333849:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982957184:0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.333881:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319995.333903:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767319995.333908:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982957952:0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.333930:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319995.333953:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957952 00000100:00100000:0.0:1767319995.333958:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957952 00000100:00100000:1.0:1767319995.333998:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905 00000100:00100000:1.0:1767319995.334009:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982957824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.334059:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982957824:12345-0@lo:400:kworker.0 Request processed in 52us (846us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319995.334127:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982957312:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.334156:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.334176:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982957568:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.334195:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.334211:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982957696:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.334231:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.334246:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982958080:0@lo:38:kworker.0 00000100:00100000:3.0:1767319995.334265:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319995.334307:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957440 00000100:00100000:3.0:1767319995.334311:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957440 00000100:00100000:3.0:1767319995.334331:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957184 00000100:00100000:3.0:1767319995.334334:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957184 00000100:00100000:3.0:1767319995.334344:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957312 00000100:00100000:3.0:1767319995.334346:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957312 00000100:00100000:3.0:1767319995.334354:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957568 00000100:00100000:3.0:1767319995.334357:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957568 00000100:00100000:3.0:1767319995.334364:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958080 00000100:00100000:3.0:1767319995.334366:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958080 00000100:00100000:3.0:1767319995.334375:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1448 00000100:00100000:3.0:1767319995.334381:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982957440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.334425:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982957440:12345-0@lo:400:kworker.0 Request processed in 44us (1111us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319995.334431:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1448 00000100:00100000:3.0:1767319995.334460:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1449 00000100:00100000:3.0:1767319995.334468:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982957184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.334500:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982957184:12345-0@lo:400:kworker.0 Request processed in 34us (619us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319995.334506:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1449 00000100:00100000:3.0:1767319995.334532:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912 00000100:00100000:3.0:1767319995.334536:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982957312:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.334556:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.334603:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982957312:12345-0@lo:38:kworker.0 Request processed in 65us (446us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.334610:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912 00000100:00100000:3.0:1767319995.334623:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1913 00000100:00100000:3.0:1767319995.334626:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982957568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.334636:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.334667:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982957568:12345-0@lo:38:kworker.0 Request processed in 38us (470us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.334672:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1913 00000100:00100000:3.0:1767319995.334682:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914 00000100:00100000:3.0:1767319995.334685:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982958080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.334693:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.334716:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982958080:12345-0@lo:38:kworker.0 Request processed in 29us (450us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.334722:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914 00000100:00100000:3.0:1767319995.334746:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982957696 00000100:00100000:3.0:1767319995.334750:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982957696 00000100:00100000:3.0:1767319995.334767:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 503 00000100:00100000:3.0:1767319995.334772:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982957696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767319995.334780:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767319995.334804:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982957696:12345-0@lo:38:kworker.0 Request processed in 32us (572us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767319995.334809:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 503 00010000:00080000:3.0:1767319995.334843:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c2a00 x1853163982957184/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320011 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319995.334858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982957184:0@lo:400:kworker.0 00000100:00080000:3.0:1767319995.334887:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.334892:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.334896:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982957312:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.334905:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.334908:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.334911:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982957568:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.334919:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.334922:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.334925:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982957696:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.334932:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767319995.334935:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767319995.334937:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982958080:0@lo:38:kworker.0 00000100:00080000:3.0:1767319995.334970:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767319995.334974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:3.0:1767319995.335026:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35500 x1853163982957440/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320011 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767319995.335036:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982957440:0@lo:400:kworker.0 00000100:00100000:3.0:1767319995.335047:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982957824:0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.346034:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 906 00000100:00100000:0.0:1767319995.346058:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982957952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767319995.346161:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982957952:12345-0@lo:400:kworker.0 Request processed in 116us (12229us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319995.346170:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 906 00000100:00100000:3.0:1767319995.348780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982957952:0@lo:400:kworker.0 00000100:00100000:1.0:1767319995.350200:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905 00010000:00010000:1.0:1767319995.524833:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319996.548766:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767319996.868875:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767319996.868896:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982958208:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319996.868965:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767319996.869031:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958208 00000100:00100000:3.0:1767319996.869038:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958208 00000100:00100000:3.0:1767319996.869093:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 976 00000100:00100000:3.0:1767319996.869102:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982958208:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767319996.869133:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767319996.869175:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982958208:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767319996.869182:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 976 00000100:00100000:3.0:1767319996.869230:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982958208:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767319996.871586:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767319996.871602:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982958336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767319996.871651:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767319996.871716:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958336 00000100:00100000:0.0:1767319996.871722:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958336 00000100:00100000:0.0:1767319996.871764:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977 00000100:00100000:0.0:1767319996.871782:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982958336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767319996.871809:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767319996.871855:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982958336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (203us total) trans 0 rc 0/0 00000100:00100000:0.0:1767319996.871863:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977 00000100:00100000:1.0:1767319996.871899:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982958336:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767319997.324691:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767319997.324836:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767319997.331794:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656576 00000100:00100000:3.0:1767319997.331814:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656576 00000100:00100000:2.0:1767319997.332059:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656448 00000100:00100000:2.0:1767319997.332065:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656448 00000100:00100000:2.0:1767319997.332106:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1450 00000100:00100000:2.0:1767319997.332117:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915656576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767319997.332190:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1915 00000100:00100000:0.0:1767319997.332209:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915656448:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767319997.332229:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915656576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 113us (7548us total) trans 0 rc 0/0 00000100:00100000:2.0:1767319997.332238:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1450 00010000:00020000:0.0:1767319997.332243:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767319997.332303:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915656448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 94us (7467us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767319997.332311:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1915 00000100:00080000:3.0:1767319997.332469:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767319997.572549:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767319998.596757:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767319999.620507:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320000.452586:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.452597:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320000.452662:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.452667:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.452670:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.452678:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.452683:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.452688:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320000.452712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982958592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320000.452760:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.452763:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.452765:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.452768:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.452771:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.452775:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320000.452787:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982958848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320000.452796:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.452799:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.452800:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.452804:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.452806:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.452809:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320000.452819:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982958976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320000.452842:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320000.452845:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320000.452847:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320000.452850:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5243 00000100:00080000:1.0:1767320000.452852:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320000.452855:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320000.452864:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0e00 x1853163982959360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320000.452928:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982959232:0@lo:400:kworker.0 00000100:00100000:3.0:1767320000.452934:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982958720:0@lo:400:kworker.0 00000100:00100000:0.0:1767320000.453011:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982958592:0@lo:38:kworker.0 00000100:00100000:2.0:1767320000.454445:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320000.454446:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320000.454492:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320000.454502:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982958464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.454530:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320000.454702:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982959104:0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.454728:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320000.454760:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958720 00000100:00100000:2.0:1767320000.454769:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958720 00000100:00100000:2.0:1767320000.454822:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958464 00000100:00100000:2.0:1767320000.454825:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958464 00000100:00100000:2.0:1767320000.454848:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1451 00000100:00100000:2.0:1767320000.454856:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982958720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.454904:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982958720:12345-0@lo:400:kworker.0 Request processed in 48us (463us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320000.454911:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1451 00000100:00100000:2.0:1767320000.454926:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1452 00000100:00100000:2.0:1767320000.454932:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982958464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.454959:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982958464:12345-0@lo:400:kworker.0 Request processed in 27us (428us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320000.455124:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1452 00000100:00100000:2.0:1767320000.455146:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959232 00000100:00100000:2.0:1767320000.455150:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959232 00000100:00100000:2.0:1767320000.455176:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 907 00000100:00100000:2.0:1767320000.455181:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982959232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.455214:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982959232:12345-0@lo:400:kworker.0 Request processed in 33us (776us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320000.455220:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 907 00000100:00100000:2.0:1767320000.455228:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959104 00000100:00100000:2.0:1767320000.455231:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959104 00000100:00100000:2.0:1767320000.455254:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908 00000100:00100000:2.0:1767320000.455259:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982959104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320000.455314:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982959104:12345-0@lo:400:kworker.0 Request processed in 55us (585us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320000.455320:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908 00000100:00080000:2.0:1767320000.455335:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320000.455340:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320000.455342:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320000.455463:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4000 x1853163982958720/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320016 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320000.455486:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982958720:0@lo:400:kworker.0 00000100:00100000:1.0:1767320000.459681:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982959104:0@lo:400:kworker.0 00010000:00080000:1.0:1767320000.459731:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6d80 x1853163982958464/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320016 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320000.460130:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982958464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320000.460148:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982959232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320000.461306:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.461342:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982958848:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.461386:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.461412:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982958976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.461435:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.461486:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982959360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320000.461512:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320000.461562:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958592 00000100:00100000:0.0:1767320000.461569:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958592 00000100:00100000:0.0:1767320000.461600:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958848 00000100:00100000:0.0:1767320000.461603:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958848 00000100:00100000:0.0:1767320000.461615:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959360 00000100:00100000:0.0:1767320000.461617:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959360 00000100:00100000:0.0:1767320000.461630:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1916 00000100:00100000:0.0:1767320000.461638:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982958592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320000.461658:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.461698:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982958592:12345-0@lo:38:kworker.0 Request processed in 60us (395us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.461704:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1916 00000100:00100000:0.0:1767320000.461720:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917 00000100:00100000:0.0:1767320000.461724:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982958848:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320000.461732:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.461891:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982958848:12345-0@lo:38:kworker.0 Request processed in 165us (503us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.461897:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917 00000100:00100000:0.0:1767320000.461908:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918 00000100:00100000:0.0:1767320000.461912:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982959360:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320000.461921:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320000.461937:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982959360:12345-0@lo:38:kworker.0 Request processed in 24us (424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320000.461942:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918 00000100:00080000:0.0:1767320000.461973:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.461978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.461981:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982958592:0@lo:38:kworker.0 00000100:00080000:0.0:1767320000.461988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.461991:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.461993:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982958848:0@lo:38:kworker.0 00000100:00080000:0.0:1767320000.462000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.462003:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.462006:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982959360:0@lo:38:kworker.0 00000100:00100000:3.0:1767320000.462232:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982958976 00000100:00100000:3.0:1767320000.462237:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982958976 00000100:00100000:3.0:1767320000.462266:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 504 00000100:00100000:3.0:1767320000.462289:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982958976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320000.462299:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320000.462331:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982958976:12345-0@lo:38:kworker.0 Request processed in 41us (894us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320000.462337:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 504 00000100:00080000:0.0:1767320000.462623:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320000.462628:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320000.462631:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982958976:0@lo:38:kworker.0 00010000:00010000:2.0:1767320000.644515:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320001.668718:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320001.988575:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982959488:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320001.988665:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320001.989116:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320001.989295:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959488 00000100:00100000:1.0:1767320001.989304:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959488 00000100:00100000:1.0:1767320001.989744:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 978 00000100:00100000:1.0:1767320001.989755:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982959488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320001.989789:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320001.989855:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982959488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (1190us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320001.989862:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 978 00000100:00100000:3.0:1767320001.989942:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982959488:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320001.992625:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982959616:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320001.993073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320001.997640:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959616 00000100:00100000:2.0:1767320001.997657:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959616 00000100:00100000:2.0:1767320001.997732:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 979 00000100:00100000:2.0:1767320001.997743:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982959616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320001.997781:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320001.997874:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982959616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (4800us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320001.997881:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 979 00000100:00100000:0.0:1767320001.998010:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982959616:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320002.444575:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320002.444762:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320002.445516:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656832 00000100:00100000:1.0:1767320002.445532:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656832 00000100:00100000:0.0:1767320002.445663:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656704 00000100:00100000:0.0:1767320002.445669:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656704 00000100:00100000:0.0:1767320002.445716:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1453 00000100:00100000:0.0:1767320002.445731:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915656832:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320002.445748:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1919 00000100:00100000:3.0:1767320002.445758:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915656704:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320002.445784:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320002.445853:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915656832:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 125us (1292us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320002.445853:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915656704:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 93us (1088us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320002.445870:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1919 00000100:00080000:2.0:1767320002.446017:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320002.447576:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1453 00010000:00010000:2.0:1767320002.692543:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320003.716590:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320004.740627:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320005.572906:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.572922:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320005.573020:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.573026:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.573031:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.573042:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.573049:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.573060:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320005.573095:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320005.573101:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1f80 x1853163982959872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320005.573131:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982959744:0@lo:400:kworker.0 00000100:00080000:1.0:1767320005.573166:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.573171:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.573173:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.573178:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.573181:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.573186:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320005.573200:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1c00 x1853163982960128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320005.573211:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.573215:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.573216:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.573220:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.573223:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.573228:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320005.573239:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320005.573246:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0e00 x1853163982960256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320005.573329:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320005.573333:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320005.573335:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320005.573339:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5248 00000100:00080000:1.0:1767320005.573342:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320005.573347:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320005.573362:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016300 x1853163982960640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320005.573381:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982960512:0@lo:400:kworker.0 00000100:00100000:1.0:1767320005.573393:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982960000:0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.573419:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320005.573436:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320005.573471:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320005.573471:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960512 00000100:00100000:1.0:1767320005.573480:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982960384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.573483:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960512 00000100:00100000:1.0:1767320005.573509:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320005.573544:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959744 00000100:00100000:1.0:1767320005.573549:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959744 00000100:00100000:3.0:1767320005.573561:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960384 00000100:00100000:3.0:1767320005.573567:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960384 00000100:00100000:3.0:1767320005.573599:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909 00000100:00100000:1.0:1767320005.573605:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960000 00000100:00100000:1.0:1767320005.573611:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960000 00000100:00100000:3.0:1767320005.573611:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982960512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320005.573642:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1454 00000100:00100000:1.0:1767320005.573651:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982959744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320005.573684:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982960512:12345-0@lo:400:kworker.0 Request processed in 74us (263us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320005.573693:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909 00000100:00100000:3.0:1767320005.573714:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 910 00000100:00100000:3.0:1767320005.573720:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982960384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320005.573812:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1455 00000100:00100000:0.0:1767320005.573826:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982960000:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320005.574318:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320005.574326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320005.574329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320005.574349:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982960384:12345-0@lo:400:kworker.0 Request processed in 626us (837us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320005.574359:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982959872:0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574362:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 910 00000100:00100000:2.0:1767320005.574417:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320005.574430:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982960512:0@lo:400:kworker.0 00010000:00080000:3.0:1767320005.574450:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f0380 x1853163982959744/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320021 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320005.574457:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982960128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574467:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982959744:0@lo:400:kworker.0 00000100:00100000:2.0:1767320005.574489:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320005.574508:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982960256:0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574532:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982959872 00000100:00100000:2.0:1767320005.574537:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320005.574539:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982959872 00000100:00100000:3.0:1767320005.574558:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960128 00000100:00100000:3.0:1767320005.574561:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960128 00000100:00100000:2.0:1767320005.574570:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982960640:0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574576:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920 00000100:00100000:3.0:1767320005.574583:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982959872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320005.574605:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320005.574613:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320005.574643:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960256 00000100:00100000:2.0:1767320005.574648:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960256 00000100:00100000:3.0:1767320005.574656:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982959872:12345-0@lo:38:kworker.0 Request processed in 71us (238us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320005.574663:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920 00000100:00100000:3.0:1767320005.574677:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960640 00000100:00100000:2.0:1767320005.574679:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 505 00000100:00100000:3.0:1767320005.574682:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960640 00000100:00100000:2.0:1767320005.574688:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982960256:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574697:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1921 00000100:00100000:3.0:1767320005.574703:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982960128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320005.574714:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320005.574718:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320005.574759:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982960256:12345-0@lo:38:kworker.0 Request processed in 70us (223us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320005.574762:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982960128:12345-0@lo:38:kworker.0 Request processed in 57us (273us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320005.574773:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 505 00000100:00100000:3.0:1767320005.574777:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1921 00000100:00100000:3.0:1767320005.574794:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1922 00000100:00100000:3.0:1767320005.574799:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982960640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320005.574810:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320005.574817:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.574825:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.574829:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982959872:0@lo:38:kworker.0 00000100:00100000:3.0:1767320005.574844:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982960640:12345-0@lo:38:kworker.0 Request processed in 43us (230us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767320005.574846:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.574853:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320005.574857:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1922 00000100:00100000:2.0:1767320005.574860:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982960128:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.574874:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.574879:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.574882:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982960256:0@lo:38:kworker.0 00000100:00080000:2.0:1767320005.574891:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320005.574894:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320005.574897:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982960640:0@lo:38:kworker.0 00000100:00100000:0.0:1767320005.575399:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982960000:12345-0@lo:400:kworker.0 Request processed in 1574us (1960us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320005.575410:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1455 00000100:00100000:3.0:1767320005.575569:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982960384:0@lo:400:kworker.0 00010000:00080000:3.0:1767320005.575583:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2d80 x1853163982960000/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320021 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320005.575594:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982960000:0@lo:400:kworker.0 00000100:00100000:1.0:1767320005.579920:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982959744:12345-0@lo:400:kworker.0 Request processed in 6241us (6653us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320005.579943:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1454 00010000:00010000:2.0:1767320005.764577:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320006.790934:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320007.108757:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320007.109014:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982960768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320007.109116:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320007.109256:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982960896:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320007.109303:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320007.109418:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960768 00000100:00100000:1.0:1767320007.109454:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960768 00000100:00100000:1.0:1767320007.109555:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982960896 00000100:00100000:1.0:1767320007.109559:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982960896 00000100:00100000:1.0:1767320007.109583:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980 00000100:00100000:1.0:1767320007.109592:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982960768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320007.109633:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320007.109692:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982960768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 101us (576us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320007.109701:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980 00000100:00100000:1.0:1767320007.109723:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981 00000100:00100000:1.0:1767320007.109730:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982960896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320007.109748:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320007.109783:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982960896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (479us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320007.109789:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981 00000100:00100000:1.0:1767320007.109843:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982960896:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320007.110120:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320007.110126:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320007.110154:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982960768:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320007.565747:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320007.565879:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657088 00000100:00100000:1.0:1767320007.565893:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657088 00000100:00100000:1.0:1767320007.566000:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1456 00000100:00100000:1.0:1767320007.566014:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915657088:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320007.566126:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915657088:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (508us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320007.566135:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1456 00000100:00080000:3.0:1767320007.566235:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320007.572072:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320007.573751:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915656960 00000100:00100000:1.0:1767320007.573774:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915656960 00000100:00100000:1.0:1767320007.573862:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1923 00000100:00100000:1.0:1767320007.573873:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915656960:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320007.573903:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320007.574011:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915656960:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 138us (1955us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320007.574021:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1923 00010000:00010000:1.0:1767320007.812578:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320008.837755:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320009.861031:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320010.692550:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.692565:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320010.692655:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.692661:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.692666:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.692677:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.692683:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.692693:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320010.692726:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163982961152/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320010.692781:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.692785:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.692787:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.692792:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.692794:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.692799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320010.692812:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015500 x1853163982961408/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320010.692821:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982961024:0@lo:400:kworker.0 00000100:00080000:1.0:1767320010.692821:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.692824:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.692826:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.692830:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.692832:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.692835:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320010.692853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398f480 x1853163982961536/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320010.692878:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320010.692882:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320010.692898:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320010.692902:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5254 00000100:00080000:1.0:1767320010.692905:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320010.692908:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320010.692913:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.692920:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c000 x1853163982961920/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320010.692944:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982961664:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.692983:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320010.693007:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320010.693013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982961280:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.693035:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320010.693053:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320010.693060:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982961792:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.693070:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961024 00000100:00100000:1.0:1767320010.693078:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961024 00000100:00100000:0.0:1767320010.693082:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320010.693110:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961280 00000100:00100000:0.0:1767320010.693114:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961280 00000100:00100000:1.0:1767320010.693141:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1457 00000100:00100000:1.0:1767320010.693156:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982961024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.693225:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982961024:12345-0@lo:400:kworker.0 Request processed in 76us (310us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320010.693233:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1458 00000100:00100000:0.0:1767320010.693241:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982961280:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.693243:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1457 00000100:00100000:0.0:1767320010.693305:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982961280:12345-0@lo:400:kworker.0 Request processed in 65us (269us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320010.693515:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1458 00000100:00100000:2.0:1767320010.693515:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982961152:0@lo:38:kworker.0 00000100:00100000:2.0:1767320010.693549:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767320010.693553:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016300 x1853163982961280/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320026 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320010.693569:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982961280:0@lo:400:kworker.0 00000100:00100000:2.0:1767320010.693575:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982961408:0@lo:38:kworker.0 00000100:00100000:2.0:1767320010.693594:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320010.693612:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982961536:0@lo:38:kworker.0 00000100:00100000:0.0:1767320010.693616:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961152 00000100:00100000:0.0:1767320010.693619:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961152 00000100:00100000:0.0:1767320010.693635:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961408 00000100:00100000:0.0:1767320010.693638:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961408 00000100:00100000:2.0:1767320010.693638:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320010.693660:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982961920:0@lo:38:kworker.0 00000100:00100000:2.0:1767320010.693682:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320010.693706:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961536 00000100:00100000:2.0:1767320010.693710:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961536 00000100:00100000:2.0:1767320010.693733:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 506 00000100:00100000:2.0:1767320010.693738:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982961536:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320010.693763:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320010.693794:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961920 00000100:00100000:0.0:1767320010.693798:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961920 00000100:00100000:2.0:1767320010.693802:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982961536:12345-0@lo:38:kworker.0 Request processed in 63us (163us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320010.693809:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924 00000100:00100000:2.0:1767320010.693809:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 506 00000100:00100000:0.0:1767320010.693813:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982961152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320010.693823:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320010.693830:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1925 00000100:00100000:2.0:1767320010.693834:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982961408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320010.693843:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320010.693848:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982961152:12345-0@lo:38:kworker.0 Request processed in 33us (299us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320010.693853:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924 00000100:00100000:2.0:1767320010.693861:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982961408:12345-0@lo:38:kworker.0 Request processed in 26us (267us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320010.693864:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926 00000100:00100000:2.0:1767320010.693867:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1925 00000100:00100000:0.0:1767320010.693868:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982961920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320010.693877:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320010.693897:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982961920:12345-0@lo:38:kworker.0 Request processed in 28us (215us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767320010.693902:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320010.693903:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926 00000100:00080000:2.0:1767320010.693908:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.693911:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982961152:0@lo:38:kworker.0 00000100:00080000:2.0:1767320010.693920:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320010.693924:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.693927:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982961408:0@lo:38:kworker.0 00000100:00080000:2.0:1767320010.693934:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320010.693937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.693940:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982961536:0@lo:38:kworker.0 00000100:00080000:2.0:1767320010.693946:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320010.693950:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320010.693952:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982961920:0@lo:38:kworker.0 00010000:00080000:1.0:1767320010.694001:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014700 x1853163982961024/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320026 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320010.694012:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982961024:0@lo:400:kworker.0 00000100:00100000:3.0:1767320010.694127:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961664 00000100:00100000:3.0:1767320010.694131:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961664 00000100:00100000:3.0:1767320010.694174:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982961792 00000100:00100000:3.0:1767320010.694177:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982961792 00000100:00100000:3.0:1767320010.694203:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 911 00000100:00100000:3.0:1767320010.694211:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982961664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320010.694312:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982961664:12345-0@lo:400:kworker.0 Request processed in 102us (1328us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320010.694322:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 911 00000100:00100000:3.0:1767320010.694345:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912 00000100:00100000:3.0:1767320010.694352:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982961792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320010.694390:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982961792:12345-0@lo:400:kworker.0 Request processed in 38us (1305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320010.694396:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912 00000100:00080000:3.0:1767320010.694421:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320010.694425:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320010.694428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320010.694468:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982961792:0@lo:400:kworker.0 00000100:00100000:1.0:1767320010.695097:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982961664:0@lo:400:kworker.0 00010000:00010000:1.0:1767320010.887411:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320011.909313:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320012.228687:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982962048:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320012.228751:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320012.228865:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962048 00000100:00100000:1.0:1767320012.228873:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962048 00000100:00100000:1.0:1767320012.228937:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 982 00000100:00100000:1.0:1767320012.228946:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982962048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320012.228976:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320012.229021:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982962048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320012.229028:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 982 00000100:00080000:3.0:1767320012.229099:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320012.229151:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982962048:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320012.231724:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320012.231741:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982962176:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320012.231791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320012.231869:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962176 00000100:00100000:2.0:1767320012.231874:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962176 00000100:00100000:2.0:1767320012.231919:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983 00000100:00100000:2.0:1767320012.231924:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982962176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320012.231951:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320012.231998:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982962176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (205us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320012.232006:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983 00000100:00100000:2.0:1767320012.232053:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982962176:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320012.232225:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320012.684799:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767320012.684835:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915657216 offset 0 length 520: 4 00000100:00100000:2.0:1767320012.684856:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320012.684935:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657472 00000100:00100000:0.0:1767320012.684950:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657472 00000100:00080000:1.0:1767320012.685022:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320012.685040:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657344 00000100:00100000:0.0:1767320012.685044:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657344 00000100:00100000:0.0:1767320012.685071:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1459 00000100:00100000:0.0:1767320012.685083:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915657472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320012.685167:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1927 00000100:00100000:0.0:1767320012.685169:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915657472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320012.685186:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915657344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320012.685210:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320012.685244:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915657344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (386us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320012.686177:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1459 00000100:00100000:1.0:1767320012.688767:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1927 00010000:00010000:2.0:1767320012.932534:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320013.957539:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320014.980539:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320015.812805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320015.812816:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320015.812880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320015.812884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320015.812887:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320015.812895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5259 00000100:00080000:0.0:1767320015.812900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320015.812914:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320015.812934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fea00 x1853163982962432/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320015.812967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320015.812971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320015.812972:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320015.812976:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5259 00000100:00080000:0.0:1767320015.812978:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320015.812982:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320015.812991:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015500 x1853163982962688/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320015.813000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320015.813003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320015.813005:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320015.813008:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5259 00000100:00080000:0.0:1767320015.813010:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320015.813014:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320015.813023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163982962816/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320015.813045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320015.813048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320015.813050:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320015.813054:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5259 00000100:00080000:0.0:1767320015.813056:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320015.813060:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320015.813069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aad80 x1853163982963200/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320015.813111:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982962560:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.813112:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982962304:0@lo:400:kworker.0 00000100:00100000:2.0:1767320015.813299:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767319990/real 1767319990] req@ffff934609a35880 x1853163982955776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320015 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320015.813422:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982963072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.816712:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.816808:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962304 00000100:00100000:1.0:1767320015.816817:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962304 00000100:00100000:1.0:1767320015.816876:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1460 00000100:00100000:1.0:1767320015.816887:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982962304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.816943:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982962304:12345-0@lo:400:kworker.0 Request processed in 59us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320015.816951:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1460 00000100:00080000:1.0:1767320015.816973:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320015.817001:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe300 x1853163982962304/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320031 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320015.817014:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982962304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.817776:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982962944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320015.817818:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.817877:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962944 00000100:00100000:1.0:1767320015.817882:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962944 00000100:00100000:0.0:1767320015.817910:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.817930:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 913 00000100:00100000:1.0:1767320015.817938:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982962944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320015.817949:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962560 00000100:00100000:0.0:1767320015.817955:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962560 00000100:00100000:1.0:1767320015.817985:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982962944:12345-0@lo:400:kworker.0 Request processed in 48us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320015.817994:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1461 00000100:00100000:1.0:1767320015.817995:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 913 00000100:00100000:0.0:1767320015.818009:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982962560:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320015.818018:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320015.818024:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320015.818046:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982962944:0@lo:400:kworker.0 00000100:00100000:0.0:1767320015.818052:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982962560:12345-0@lo:400:kworker.0 Request processed in 45us (142us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320015.818059:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1461 00010000:00080000:1.0:1767320015.818109:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014700 x1853163982962560/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320031 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320015.818120:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982962560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320015.818391:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609a35880 x1853163982955776/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320015 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320015.818413:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982962432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320015.818429:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320015.818493:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963072 00000100:00100000:1.0:1767320015.818498:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963072 00000100:00100000:1.0:1767320015.818592:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 914 00000100:00100000:1.0:1767320015.818599:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982963072:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320015.818704:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320015.818810:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320015.818847:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982962688:0@lo:38:kworker.0 00000100:00100000:0.0:1767320015.818854:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962432 00000100:00100000:0.0:1767320015.818858:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962432 00000100:00100000:0.0:1767320015.818877:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1928 00000100:00100000:2.0:1767320015.818881:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320015.818885:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982962432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320015.818902:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320015.818912:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982962816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320015.818935:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982962432:12345-0@lo:38:kworker.0 Request processed in 50us (127us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320015.818941:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1928 00000100:00100000:2.0:1767320015.818948:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320015.818950:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962688 00000100:00100000:0.0:1767320015.818954:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962688 00000100:00100000:0.0:1767320015.818968:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929 00000100:00100000:0.0:1767320015.818972:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982962688:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.818976:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982962816 00010000:00020000:0.0:1767320015.818979:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320015.818983:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982962816 00000100:00100000:0.0:1767320015.818999:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982962688:12345-0@lo:38:kworker.0 Request processed in 26us (123us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320015.819004:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929 00000100:00100000:2.0:1767320015.819014:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 507 00000100:00100000:2.0:1767320015.819022:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982962816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320015.819033:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320015.819064:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982962816:12345-0@lo:38:kworker.0 Request processed in 41us (117us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320015.819080:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 507 00000100:00100000:1.0:1767320015.819100:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982963072:12345-0@lo:400:kworker.0 Request processed in 495us (666us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320015.819109:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 914 00000100:00080000:2.0:1767320015.819121:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320015.819136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320015.819139:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982962816:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.819144:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982963200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320015.819352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320015.819379:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320015.819401:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320015.819404:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982955776:0@lo:250:kworker.0 00000100:00100000:0.0:1767320015.819418:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963200 00000100:00100000:3.0:1767320015.819418:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982963072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320015.819423:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963200 00000100:00100000:0.0:1767320015.819443:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1930 00000100:00100000:0.0:1767320015.819448:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982963200:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767320015.819448:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320015.819456:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320015.819461:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982962432:0@lo:38:kworker.0 00010000:00020000:0.0:1767320015.819464:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320015.819475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320015.819480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320015.819484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982962688:0@lo:38:kworker.0 00000100:00080000:2.0:1767320015.819497:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320015.819502:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982963200:12345-0@lo:38:kworker.0 Request processed in 50us (146us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767320015.819504:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320015.819508:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982963200:0@lo:38:kworker.0 00000100:00080000:1.0:1767320015.819603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320015.819607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320015.819609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320015.819612:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5238 00000100:00080000:1.0:1767320015.819615:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320015.819619:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320015.819638:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398f480 x1853163982963328/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320015.819694:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1930 00000100:00100000:2.0:1767320015.819731:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982963328:0@lo:250:kworker.0 00000400:00000100:2.0:1767320015.819753:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982963328 offset 0 length 520: 4 00010000:00010000:2.0:1767320016.004991:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320017.028764:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320017.349122:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982963456:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320017.349215:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320017.349384:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982963584:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320017.349412:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320017.349530:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963456 00000100:00100000:2.0:1767320017.349538:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963456 00000100:00100000:2.0:1767320017.349608:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963584 00000100:00100000:2.0:1767320017.349612:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963584 00000100:00100000:2.0:1767320017.349631:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984 00000100:00100000:2.0:1767320017.349639:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982963456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320017.349671:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767320017.349747:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320017.349751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320017.349775:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 985 00000100:00100000:1.0:1767320017.349783:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982963584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320017.349801:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320017.349835:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982963584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (420us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320017.349842:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 985 00000100:00100000:1.0:1767320017.349886:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982963584:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320017.363940:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982963456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 14289us (14712us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320017.363958:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984 00000100:00100000:0.0:1767320017.364172:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982963456:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320017.806423:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320017.806470:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320017.806639:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657600 00000100:00100000:0.0:1767320017.806646:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657728 00000100:00100000:1.0:1767320017.806650:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657600 00000100:00100000:0.0:1767320017.806653:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657728 00000100:00100000:1.0:1767320017.806704:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1462 00000100:00100000:1.0:1767320017.806715:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915657728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320017.806726:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1931 00000100:00100000:0.0:1767320017.806734:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915657600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320017.806761:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320017.806822:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915657600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 87us (419us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320017.806831:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1931 00000100:00100000:1.0:1767320017.806840:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915657728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 127us (368us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320017.806849:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1462 00000100:00080000:2.0:1767320017.806929:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320018.053245:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320019.077466:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320020.102049:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320020.932470:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320020.932480:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320020.932549:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982963712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320020.932645:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320020.932700:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963712 00000100:00100000:2.0:1767320020.932712:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963712 00000100:00100000:2.0:1767320020.932870:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1463 00000100:00100000:2.0:1767320020.932883:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982963712:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320020.932956:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320020.933132:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320020.933328:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320020.933333:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320020.933338:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5264 00000100:00080000:3.0:1767320020.933343:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320020.933347:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320020.933370:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163982963840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320020.933414:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320020.933417:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320020.933419:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320020.933424:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5264 00000100:00080000:3.0:1767320020.933426:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320020.933429:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320020.933441:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163982964096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320020.933449:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320020.933451:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320020.933453:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320020.933457:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5264 00000100:00080000:3.0:1767320020.933459:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320020.933463:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320020.933473:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163982964224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320020.933479:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982963840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.933519:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.933543:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982964096:0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.933565:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.933595:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982964224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.933618:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.933648:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963840 00000100:00100000:0.0:1767320020.933653:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963840 00000100:00100000:0.0:1767320020.933675:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964096 00000100:00100000:0.0:1767320020.933677:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964096 00000100:00100000:0.0:1767320020.933693:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1932 00000100:00100000:0.0:1767320020.933698:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982963840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320020.933718:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320020.933753:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982963840:12345-0@lo:38:kworker.0 Request processed in 53us (233us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.933759:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1932 00000100:00100000:0.0:1767320020.933774:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933 00000100:00100000:0.0:1767320020.933777:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982964096:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320020.933786:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320020.933809:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982964096:12345-0@lo:38:kworker.0 Request processed in 30us (244us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.933815:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933 00000100:00100000:0.0:1767320020.933830:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964224 00000100:00100000:0.0:1767320020.933834:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964224 00000100:00100000:0.0:1767320020.933851:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 508 00000100:00100000:0.0:1767320020.933855:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982964224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320020.933864:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320020.933887:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982964224:12345-0@lo:38:kworker.0 Request processed in 31us (268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320020.933893:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 508 00000100:00100000:0.0:1767320020.933915:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982964480:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.933937:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320020.933989:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320020.933994:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320020.933998:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982964224:0@lo:38:kworker.0 00000100:00080000:0.0:1767320020.934009:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320020.934012:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320020.934015:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982963840:0@lo:38:kworker.0 00000100:00080000:0.0:1767320020.934022:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320020.934025:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320020.934028:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982964096:0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.934056:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982964352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934079:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.934099:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964480 00000100:00100000:0.0:1767320020.934103:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964480 00000100:00100000:0.0:1767320020.934135:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964352 00000100:00100000:0.0:1767320020.934137:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964352 00000100:00100000:0.0:1767320020.934156:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915 00000100:00100000:0.0:1767320020.934162:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982964480:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934195:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982964480:12345-0@lo:400:kworker.0 Request processed in 33us (257us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320020.934200:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915 00000100:00100000:0.0:1767320020.934212:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916 00000100:00100000:0.0:1767320020.934218:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982964352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934242:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982964352:12345-0@lo:400:kworker.0 Request processed in 25us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320020.934247:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916 00000100:00080000:0.0:1767320020.934551:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320020.934561:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320020.934597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982964480:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934624:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982964352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934633:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320020.934638:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982963968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934663:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320020.934681:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982963968 00000100:00100000:0.0:1767320020.934684:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982963968 00000100:00100000:0.0:1767320020.934711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1464 00000100:00100000:0.0:1767320020.934717:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982963968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320020.934745:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982963968:12345-0@lo:400:kworker.0 Request processed in 29us (82us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320020.934751:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1464 00000100:00080000:0.0:1767320020.934765:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320020.934783:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d5880 x1853163982963968/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320036 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320020.934793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982963968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320020.935061:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982963712:12345-0@lo:400:kworker.0 Request processed in 2179us (2412us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320020.935069:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1463 00000100:00080000:3.0:1767320020.935081:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320020.935088:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320020.935091:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320020.935097:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5264 00000100:00080000:3.0:1767320020.935101:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320020.935105:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320020.935126:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163982964608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320020.935396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982964608:0@lo:38:kworker.0 00000100:00100000:0.0:1767320020.935432:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767320020.935459:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1f80 x1853163982963712/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320036 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320020.935472:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982963712:0@lo:400:kworker.0 00000100:00100000:2.0:1767320020.935601:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964608 00000100:00100000:2.0:1767320020.935606:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964608 00000100:00100000:2.0:1767320020.935635:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1934 00000100:00100000:2.0:1767320020.935640:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982964608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320020.935654:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320020.935700:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982964608:12345-0@lo:38:kworker.0 Request processed in 58us (268us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320020.935706:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1934 00000100:00080000:3.0:1767320020.935810:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320020.935815:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320020.935818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982964608:0@lo:38:kworker.0 00010000:00010000:2.0:1767320021.124981:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320022.148509:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320022.468586:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982964736:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320022.468665:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320022.468959:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320022.468970:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982964864:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320022.469037:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320022.469989:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964736 00000100:00100000:1.0:1767320022.470002:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964736 00000100:00100000:1.0:1767320022.470118:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964864 00000100:00100000:1.0:1767320022.470122:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964864 00000100:00100000:2.0:1767320022.470172:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986 00000100:00100000:2.0:1767320022.470185:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982964736:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320022.470227:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320022.470299:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982964736:12345-0@lo:13:osp-pre-0-1.0 Request processed in 117us (1633us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320022.470307:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986 00000100:00100000:2.0:1767320022.470356:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982964736:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320022.470941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320022.470957:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320022.471058:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987 00000100:00100000:1.0:1767320022.471067:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982964864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320022.471084:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320022.471131:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982964864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (2094us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320022.471137:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987 00000100:00100000:2.0:1767320022.471169:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982964864:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320022.925799:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320022.925846:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320022.925896:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657856 00000100:00100000:1.0:1767320022.925906:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657856 00000100:00100000:1.0:1767320022.926021:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915657984 00000100:00100000:1.0:1767320022.926025:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915657984 00000100:00100000:3.0:1767320022.926201:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1935 00000100:00100000:3.0:1767320022.926212:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915657856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320022.926238:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320022.926413:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1465 00000100:00100000:1.0:1767320022.926422:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915657984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320022.931351:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915657856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 5125us (5547us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320022.931367:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1935 00000100:00080000:3.0:1767320022.931588:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320022.931652:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915657984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 5229us (5802us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320022.931950:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1465 00010000:00010000:1.0:1767320023.172587:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320024.196572:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320025.220505:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320026.052485:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320026.052497:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320026.052565:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320026.052569:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320026.052572:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320026.052580:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5269 00000100:00080000:0.0:1767320026.052585:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320026.052591:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320026.052626:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6680 x1853163982965120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320026.052667:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320026.052671:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320026.052673:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320026.052677:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5269 00000100:00080000:0.0:1767320026.052679:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320026.052686:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320026.052687:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982964992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320026.052701:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7800 x1853163982965376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320026.052711:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320026.052714:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320026.052716:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320026.052719:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5269 00000100:00080000:0.0:1767320026.052723:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1767320026.052726:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320026.052739:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6a00 x1853163982965504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320026.052758:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320026.052783:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320026.052787:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320026.052789:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320026.052792:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5269 00000100:00080000:0.0:1767320026.052795:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00100000:2.0:1767320026.052797:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:0.0:1767320026.052800:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320026.052807:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982965760:0@lo:400:kworker.0 00000100:00100000:0.0:1767320026.052823:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2a00 x1853163982965888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320026.052835:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320026.052848:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982965248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320026.052872:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982964992 00000100:00100000:0.0:1767320026.052892:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320026.052908:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982964992 00000100:00100000:0.0:1767320026.052916:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320026.052923:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982965632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320026.052959:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965248 00000100:00100000:2.0:1767320026.052963:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965248 00000100:00100000:2.0:1767320026.052985:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1466 00000100:00100000:2.0:1767320026.052994:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982964992:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320026.053000:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320026.053048:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982964992:12345-0@lo:400:kworker.0 Request processed in 55us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320026.053051:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965760 00000100:00100000:0.0:1767320026.053057:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965760 00000100:00100000:2.0:1767320026.053061:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1466 00000100:00100000:2.0:1767320026.053081:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1467 00000100:00100000:2.0:1767320026.053088:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982965248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320026.053094:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965632 00000100:00100000:0.0:1767320026.053099:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965632 00000100:00100000:2.0:1767320026.053123:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982965248:12345-0@lo:400:kworker.0 Request processed in 36us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320026.053129:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917 00000100:00100000:2.0:1767320026.053136:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1467 00000100:00100000:0.0:1767320026.053141:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982965760:12345-0@lo:400:kworker.0 00010000:00080000:2.0:1767320026.053172:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4380 x1853163982964992/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320042 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320026.053184:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982964992:0@lo:400:kworker.0 00000100:00100000:2.0:1767320026.053223:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 918 00000100:00100000:2.0:1767320026.053230:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982965632:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320026.053500:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982965760:12345-0@lo:400:kworker.0 Request processed in 362us (665us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320026.053508:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917 00000100:00080000:3.0:1767320026.053775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320026.053780:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320026.053782:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320026.053799:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982965120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.053847:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982965632:12345-0@lo:400:kworker.0 Request processed in 616us (846us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320026.053860:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 918 00000100:00100000:3.0:1767320026.053860:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320026.053906:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982965376:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.053910:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982965760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320026.053934:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320026.053950:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982965504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.053957:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965120 00000100:00100000:2.0:1767320026.053962:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965120 00000100:00100000:3.0:1767320026.053974:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320026.053985:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965376 00000100:00100000:2.0:1767320026.053989:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965376 00000100:00100000:2.0:1767320026.054014:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936 00000100:00100000:3.0:1767320026.054018:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982965888:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.054023:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965120:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320026.054043:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320026.054055:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320026.054084:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965120:12345-0@lo:38:kworker.0 Request processed in 62us (229us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320026.054091:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936 00000100:00080000:3.0:1767320026.054092:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320026.054099:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320026.054103:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965888 00000100:00100000:3.0:1767320026.054104:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982965120:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.054108:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965888 00000100:00100000:2.0:1767320026.054126:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1937 00000100:00100000:2.0:1767320026.054132:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965376:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320026.054137:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982965504 00000100:00100000:3.0:1767320026.054142:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982965504 00010000:00020000:2.0:1767320026.054143:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320026.054170:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 509 00000100:00100000:3.0:1767320026.054177:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982965504:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.054182:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965376:12345-0@lo:38:kworker.0 Request processed in 46us (245us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767320026.054191:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320026.054192:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1937 00000100:00100000:2.0:1767320026.054207:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938 00000100:00100000:2.0:1767320026.054213:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965888:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320026.054225:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320026.054226:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982965504:12345-0@lo:38:kworker.0 Request processed in 47us (249us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320026.054234:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 509 00000100:00080000:3.0:1767320026.054262:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320026.054266:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320026.054290:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982965376:0@lo:38:kworker.0 00000100:00080000:3.0:1767320026.054299:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320026.054303:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320026.054305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982965504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.054429:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982965888:12345-0@lo:38:kworker.0 Request processed in 214us (373us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767320026.054533:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320026.054537:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320026.054540:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982965888:0@lo:38:kworker.0 00000100:00100000:2.0:1767320026.054624:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938 00010000:00080000:1.0:1767320026.056252:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4a80 x1853163982965248/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320042 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320026.056281:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982965248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320026.056310:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982965632:0@lo:400:kworker.0 00010000:00010000:1.0:1767320026.244546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320027.272032:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320027.590624:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320027.590659:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982966016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320027.590732:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320027.590949:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966016 00000100:00100000:1.0:1767320027.590957:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966016 00000100:00100000:1.0:1767320027.591009:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988 00000100:00100000:1.0:1767320027.591018:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982966016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320027.591046:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320027.591097:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982966016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (365us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320027.591105:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988 00000100:00100000:0.0:1767320027.591225:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982966016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320027.593256:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320027.593289:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982966144:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320027.593348:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320027.593455:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966144 00000100:00100000:2.0:1767320027.593464:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966144 00000100:00100000:2.0:1767320027.593515:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989 00000100:00100000:2.0:1767320027.593524:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982966144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320027.593551:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320027.593602:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982966144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (252us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320027.593610:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989 00000100:00100000:0.0:1767320027.593699:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982966144:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320028.044453:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320028.044548:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658240 00000100:00100000:0.0:1767320028.044559:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658240 00000100:00100000:0.0:1767320028.044632:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1468 00000100:00100000:0.0:1767320028.044642:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915658240:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320028.044726:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915658240:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320028.044734:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1468 00000100:00080000:3.0:1767320028.045031:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320028.045536:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320028.045662:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658112 00000100:00100000:2.0:1767320028.045669:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658112 00000100:00100000:2.0:1767320028.045699:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1939 00000100:00100000:2.0:1767320028.045706:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915658112:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320028.045731:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320028.045786:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915658112:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 79us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320028.045793:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1939 00010000:00010000:2.0:1767320028.292615:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320029.322212:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320030.340531:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320031.172460:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320031.172474:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320031.172537:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320031.172541:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320031.172546:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320031.172556:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767320031.172556:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5274 00000100:00080000:3.0:1767320031.172561:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320031.172567:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320031.172585:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982966272:0@lo:400:kworker.0 00000100:00100000:3.0:1767320031.172597:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ed80 x1853163982966400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320031.172639:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320031.172643:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320031.172645:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320031.172650:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5274 00000100:00080000:3.0:1767320031.172652:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320031.172653:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767320031.172658:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320031.172669:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982966528:0@lo:400:kworker.0 00000100:00100000:3.0:1767320031.172675:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ea00 x1853163982966656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320031.172677:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320031.172689:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320031.172693:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320031.172695:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320031.172699:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5274 00000100:00080000:3.0:1767320031.172703:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320031.172706:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320031.172719:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163982966784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320031.172740:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320031.172743:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320031.172745:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320031.172748:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5274 00000100:00080000:3.0:1767320031.172752:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320031.172755:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320031.172764:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163982967168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320031.172825:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982966400:0@lo:38:kworker.0 00000100:00100000:0.0:1767320031.173405:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320031.173419:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982966912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320031.179050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320031.179244:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320031.179265:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982967040:0@lo:400:kworker.0 00000100:00100000:2.0:1767320031.179340:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320031.179377:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967040 00000100:00100000:2.0:1767320031.179386:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967040 00000100:00100000:2.0:1767320031.179457:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 919 00000100:00100000:2.0:1767320031.179466:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982967040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320031.179530:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982967040:12345-0@lo:400:kworker.0 Request processed in 65us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.179538:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 919 00000100:00100000:2.0:1767320031.179577:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982967040:0@lo:400:kworker.0 00000100:00100000:2.0:1767320031.179633:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966272 00000100:00100000:2.0:1767320031.179638:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966272 00000100:00100000:2.0:1767320031.179670:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966528 00000100:00100000:2.0:1767320031.179673:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966528 00000100:00100000:1.0:1767320031.179689:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1469 00000100:00100000:1.0:1767320031.179700:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982966272:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320031.179744:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982966272:12345-0@lo:400:kworker.0 Request processed in 47us (7071us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.181622:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1470 00000100:00100000:2.0:1767320031.182081:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982966528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320031.182550:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982966528:12345-0@lo:400:kworker.0 Request processed in 916us (3508us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320031.182564:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1470 00000100:00100000:3.0:1767320031.182566:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320031.182590:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966400 00000100:00100000:2.0:1767320031.182598:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966400 00000100:00100000:3.0:1767320031.182612:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982966656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320031.182623:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1940 00000100:00100000:2.0:1767320031.182630:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982966400:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320031.182644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320031.183078:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320031.183116:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982966400:12345-0@lo:38:kworker.0 Request processed in 484us (553us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320031.183123:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1940 00010000:00080000:2.0:1767320031.183261:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543d180 x1853163982966528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320047 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320031.183293:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982966528:0@lo:400:kworker.0 00000100:00100000:1.0:1767320031.183478:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1469 00010000:00080000:1.0:1767320031.183519:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c2a00 x1853163982966272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320047 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320031.183530:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982966272:0@lo:400:kworker.0 00000100:00080000:1.0:1767320031.183733:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320031.183741:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320031.183895:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320031.184027:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966912 00000100:00100000:1.0:1767320031.184032:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966912 00000100:00100000:1.0:1767320031.184186:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767320031.184193:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982966912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320031.184241:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982966912:12345-0@lo:400:kworker.0 Request processed in 49us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320031.184247:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920 00000100:00100000:1.0:1767320031.184394:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982966912:0@lo:400:kworker.0 00000100:00080000:0.0:1767320031.184495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320031.185622:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982966784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320031.185735:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320031.185775:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982967168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320031.185816:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320031.185867:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320031.185874:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320031.185878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982966400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320031.185884:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966656 00000100:00100000:1.0:1767320031.185892:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966656 00000100:00100000:1.0:1767320031.185927:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967168 00000100:00100000:1.0:1767320031.185930:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967168 00000100:00100000:3.0:1767320031.185938:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941 00000100:00100000:3.0:1767320031.186311:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982966656:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320031.186322:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942 00000100:00100000:1.0:1767320031.186329:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982967168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320031.186334:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320031.186344:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320031.186393:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982967168:12345-0@lo:38:kworker.0 Request processed in 63us (575us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320031.186400:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942 00000100:00100000:3.0:1767320031.186463:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982966656:12345-0@lo:38:kworker.0 Request processed in 516us (3818us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320031.186470:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941 00000100:00100000:0.0:1767320031.186554:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982966784 00000100:00100000:0.0:1767320031.186560:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982966784 00000100:00100000:0.0:1767320031.186584:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 510 00000100:00100000:0.0:1767320031.186590:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982966784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320031.186601:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320031.186670:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320031.186677:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320031.186680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982966656:0@lo:38:kworker.0 00000100:00100000:0.0:1767320031.191873:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982966784:12345-0@lo:38:kworker.0 Request processed in 5267us (6123us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320031.191890:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 510 00000100:00080000:2.0:1767320031.194212:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320031.194224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320031.194230:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982967168:0@lo:38:kworker.0 00000100:00080000:2.0:1767320031.194266:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320031.194285:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320031.194288:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982966784:0@lo:38:kworker.0 00010000:00010000:2.0:1767320031.364901:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320032.389306:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320032.712152:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982967296:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320032.712251:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320032.712305:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982967424:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320032.712622:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967296 00000100:00100000:3.0:1767320032.712630:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967296 00000100:00100000:3.0:1767320032.712684:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990 00000100:00100000:3.0:1767320032.712693:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982967296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320032.712745:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320032.712810:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982967296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (560us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320032.712816:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990 00000100:00100000:0.0:1767320032.712858:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982967296:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320032.712894:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320032.712920:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967424 00000100:00100000:0.0:1767320032.712924:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967424 00000100:00100000:0.0:1767320032.713026:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767320032.713033:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982967424:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320032.713064:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320032.713099:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982967424:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (205us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320032.713106:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767320032.713135:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982967424:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320033.164119:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320033.164472:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658496 00000100:00100000:3.0:1767320033.164484:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658496 00000100:00100000:3.0:1767320033.164582:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1471 00000100:00100000:3.0:1767320033.164593:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915658496:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320033.164683:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915658496:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (574us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320033.164692:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1471 00000100:00080000:0.0:1767320033.164990:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320033.165554:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320033.168223:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658368 00000100:00100000:2.0:1767320033.168235:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658368 00000100:00100000:2.0:1767320033.168295:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1943 00000100:00100000:2.0:1767320033.168302:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915658368:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320033.168322:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320033.168385:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915658368:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 84us (2835us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320033.168390:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1943 00010000:00010000:2.0:1767320033.413937:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320034.436596:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320035.460486:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320036.292384:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320036.292394:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320036.292502:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320036.292506:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320036.292510:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320036.292517:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5279 00000100:00080000:3.0:1767320036.292524:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767320036.292530:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320036.292549:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909500 x1853163982967680/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320036.292577:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320036.292581:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320036.292583:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320036.292587:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5279 00000100:00080000:3.0:1767320036.292589:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00100000:2.0:1767320036.292595:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982967808:0@lo:400:kworker.0 00000100:00080000:3.0:1767320036.292595:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320036.292610:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b100 x1853163982967936/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320036.292619:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320036.292622:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320036.292624:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320036.292627:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5279 00000100:00080000:3.0:1767320036.292630:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320036.292634:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320036.292648:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909880 x1853163982968064/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320036.292672:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320036.292674:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320036.292676:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320036.292680:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5279 00000100:00080000:3.0:1767320036.292682:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320036.292685:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320036.292709:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320036.292727:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190a680 x1853163982968448/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320036.292775:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967808 00000100:00100000:3.0:1767320036.292783:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967808 00000100:00100000:2.0:1767320036.292787:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320036.292795:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982968192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320036.292828:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320036.292864:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968192 00000100:00100000:3.0:1767320036.292866:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1472 00000100:00100000:2.0:1767320036.292871:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968192 00000100:00100000:3.0:1767320036.292882:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982967808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320036.292913:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921 00000100:00100000:2.0:1767320036.292922:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982968192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320036.292957:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982967808:12345-0@lo:400:kworker.0 Request processed in 79us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320036.292965:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1472 00000100:00100000:2.0:1767320036.292970:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982968192:12345-0@lo:400:kworker.0 Request processed in 48us (140us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320036.292978:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921 00000100:00100000:2.0:1767320036.293046:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982968192:0@lo:400:kworker.0 00010000:00080000:2.0:1767320036.293061:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190ad80 x1853163982967808/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320052 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320036.293071:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982967808:0@lo:400:kworker.0 00000100:00080000:0.0:1767320036.293176:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320036.293182:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320036.293195:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982968320:0@lo:400:kworker.0 00000100:00100000:0.0:1767320036.293232:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320036.293326:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320036.293333:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982967552:0@lo:400:kworker.0 00000100:00100000:0.0:1767320036.293374:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320036.293418:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982967680:0@lo:38:kworker.0 00000100:00100000:0.0:1767320036.293446:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320036.293474:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968320 00000100:00100000:0.0:1767320036.293480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982967936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320036.293481:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968320 00000100:00100000:0.0:1767320036.293511:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320036.293534:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 922 00000100:00100000:3.0:1767320036.293544:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967552 00000100:00100000:1.0:1767320036.293547:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982968320:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320036.293550:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967552 00000100:00100000:0.0:1767320036.293552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982968064:0@lo:38:kworker.0 00000100:00100000:3.0:1767320036.293589:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967680 00000100:00100000:3.0:1767320036.293594:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967680 00000100:00100000:0.0:1767320036.293598:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320036.293606:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982968320:12345-0@lo:400:kworker.0 Request processed in 63us (372us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320036.293615:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 922 00000100:00100000:3.0:1767320036.293615:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982967936 00000100:00100000:3.0:1767320036.293620:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982967936 00000100:00100000:0.0:1767320036.293633:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982968448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320036.293640:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1473 00000100:00100000:1.0:1767320036.293650:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:3.0:1767320036.293652:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982967552:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320036.293659:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982967680:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320036.293663:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767320036.293681:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320036.293708:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982967552:12345-0@lo:400:kworker.0 Request processed in 58us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320036.293711:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968448 00000100:00100000:0.0:1767320036.293717:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968448 00000100:00100000:3.0:1767320036.293720:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1473 00000100:00100000:1.0:1767320036.293738:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982967680:12345-0@lo:38:kworker.0 Request processed in 78us (289us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320036.293743:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1945 00000100:00100000:3.0:1767320036.293772:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1767320036.293777:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982967936:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320036.293782:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982968448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320036.293792:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320036.293797:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320036.293829:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982967936:12345-0@lo:38:kworker.0 Request processed in 77us (317us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320036.293839:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1945 00000100:00100000:3.0:1767320036.293839:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982968448:12345-0@lo:38:kworker.0 Request processed in 57us (176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320036.293852:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1767320036.293867:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968064 00000100:00100000:0.0:1767320036.293872:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968064 00000100:00080000:3.0:1767320036.293889:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320036.293894:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320036.293900:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 511 00000100:00100000:0.0:1767320036.293907:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982968064:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320036.293919:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:3.0:1767320036.293949:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631909f80 x1853163982967552/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320052 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320036.293953:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982968064:12345-0@lo:38:kworker.0 Request processed in 43us (355us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320036.293967:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 511 00000100:00080000:0.0:1767320036.293993:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320036.293998:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320036.294001:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982967680:0@lo:38:kworker.0 00000100:00080000:0.0:1767320036.294010:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320036.294013:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320036.294016:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982967936:0@lo:38:kworker.0 00000100:00080000:0.0:1767320036.294024:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320036.294027:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320036.294030:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982968064:0@lo:38:kworker.0 00000100:00080000:0.0:1767320036.294039:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320036.294042:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320036.294045:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982968448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320036.294120:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982967552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320036.294137:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982968320:0@lo:400:kworker.0 00000100:00100000:1.0:1767320036.300402:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00010000:00010000:2.0:1767320036.486532:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320037.261130:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320037.261204:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658880 00000100:00100000:2.0:1767320037.261216:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658880 00000100:00100000:2.0:1767320037.261392:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1474 00000100:00100000:2.0:1767320037.261401:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915658880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320037.261486:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915658880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320037.261494:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1474 00000100:00080000:3.0:1767320037.265305:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:0.0:1767320037.265390:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915658624 offset 0 length 520: 4 00000100:00100000:1.0:1767320037.270899:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320037.271343:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915658752 00000100:00100000:0.0:1767320037.271354:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915658752 00000100:00100000:0.0:1767320037.271407:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1947 00000100:00100000:0.0:1767320037.271416:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915658752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320037.271439:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320037.271561:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915658752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 143us (671us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320037.271569:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1947 00010000:00010000:2.0:1767320037.508731:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320037.828620:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320037.828642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982968576:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320037.828734:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320037.828740:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982968704:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320037.828812:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968576 00000100:00100000:1.0:1767320037.828825:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968576 00000100:00100000:1.0:1767320037.828902:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992 00000100:00100000:1.0:1767320037.828911:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982968576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320037.830695:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320037.830845:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320037.831132:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968704 00000100:00100000:3.0:1767320037.831141:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968704 00000100:00100000:3.0:1767320037.831193:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993 00000100:00100000:3.0:1767320037.831202:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982968704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320037.831232:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320037.831351:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982968704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 149us (506us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320037.831359:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993 00000100:00100000:1.0:1767320037.833292:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982968576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4359us (4536us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320037.833310:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992 00000100:00100000:2.0:1767320037.833505:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982968704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320037.833594:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982968576:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320038.541399:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320039.556711:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320040.388377:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320015/real 1767320015] req@ffff93463398f480 x1853163982963328/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320040 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320040.388409:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463398f480 x1853163982963328/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320040 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320040.388435:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.388508:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320040.388513:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982963328:0@lo:250:kworker.0 00000100:00080000:3.0:1767320040.394577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.394588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320040.394592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.394600:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5264 00000100:00080000:3.0:1767320040.394604:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.394660:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320040.394701:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909f80 x1853163982968832/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.394782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.394786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.394788:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.394792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5284 00000100:00080000:3.0:1767320040.394795:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320040.394798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320040.394813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909880 x1853163982969088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.394855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.394858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.394860:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.394863:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5284 00000100:00080000:3.0:1767320040.394865:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320040.394869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320040.394883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909500 x1853163982969344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.394890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.394893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.394894:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.394897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5284 00000100:00080000:3.0:1767320040.394899:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.394902:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320040.394912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190ad80 x1853163982969472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320040.394931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320040.394934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320040.394935:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320040.394938:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5284 00000100:00080000:3.0:1767320040.394940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320040.394943:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320040.394951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982969856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320040.394986:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982969216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320040.395174:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982969728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320040.395242:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320040.395353:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320040.395360:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982968960:0@lo:400:kworker.0 00000100:00100000:1.0:1767320040.395386:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320040.395431:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982969600:0@lo:400:kworker.0 00000100:00100000:1.0:1767320040.395455:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320040.395567:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969728 00000100:00100000:0.0:1767320040.395575:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969728 00000100:00100000:0.0:1767320040.395670:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969600 00000100:00100000:0.0:1767320040.395674:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969600 00000100:00100000:0.0:1767320040.395699:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767320040.395708:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982969728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.395752:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982969728:12345-0@lo:400:kworker.0 Request processed in 47us (511us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320040.395760:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923 00000100:00100000:0.0:1767320040.395771:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924 00000100:00100000:0.0:1767320040.395775:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982969600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.395792:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982969600:12345-0@lo:400:kworker.0 Request processed in 17us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320040.395796:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924 00000100:00100000:0.0:1767320040.395833:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982969728:0@lo:400:kworker.0 00000100:00080000:1.0:1767320040.395903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320040.395907:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320040.395941:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982969600:0@lo:400:kworker.0 00000100:00100000:3.0:1767320040.396322:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.396417:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982968960 00000100:00100000:2.0:1767320040.396423:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982968960 00000100:00100000:2.0:1767320040.396468:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969216 00000100:00100000:2.0:1767320040.396471:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969216 00000100:00100000:2.0:1767320040.396490:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1767320040.396496:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982968960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320040.396534:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982968960:12345-0@lo:400:kworker.0 Request processed in 39us (1147us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320040.396539:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1767320040.396551:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1476 00000100:00100000:2.0:1767320040.396556:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982969216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320040.396580:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982969216:12345-0@lo:400:kworker.0 Request processed in 25us (258us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320040.396585:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1476 00000100:00100000:2.0:1767320040.396604:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982968832:0@lo:250:kworker.0 00000400:00000100:2.0:1767320040.396639:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982968832 offset 0 length 520: 4 00000100:00100000:2.0:1767320040.396658:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982969088:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.396682:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.396699:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982969344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.396719:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.396733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982969472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.396769:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.396784:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982969856:0@lo:38:kworker.0 00000100:00100000:2.0:1767320040.396803:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320040.396821:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969088 00000100:00100000:2.0:1767320040.396825:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969088 00000100:00100000:2.0:1767320040.396835:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969344 00000100:00100000:2.0:1767320040.396838:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969344 00000100:00100000:2.0:1767320040.396844:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969856 00000100:00100000:2.0:1767320040.396846:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969856 00000100:00100000:2.0:1767320040.396853:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948 00000100:00100000:2.0:1767320040.396856:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969088:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.396885:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.396909:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969088:12345-0@lo:38:kworker.0 Request processed in 52us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.396913:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948 00000100:00100000:2.0:1767320040.396919:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1949 00000100:00100000:2.0:1767320040.396922:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969344:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.396927:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.396939:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969344:12345-0@lo:38:kworker.0 Request processed in 16us (220us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.396942:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1949 00000100:00100000:2.0:1767320040.396947:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950 00000100:00100000:2.0:1767320040.396949:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320040.396954:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320040.396966:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982969856:12345-0@lo:38:kworker.0 Request processed in 16us (163us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320040.396972:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950 00000100:00080000:1.0:1767320040.397119:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320040.397229:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.397235:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.397239:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982969088:0@lo:38:kworker.0 00000100:00080000:2.0:1767320040.397252:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.397255:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.397258:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982969344:0@lo:38:kworker.0 00000100:00080000:2.0:1767320040.397265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.397268:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.397296:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982969856:0@lo:38:kworker.0 00010000:00080000:3.0:1767320040.397399:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190b100 x1853163982969216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320056 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320040.397410:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982969216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320040.397493:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969472 00000100:00100000:0.0:1767320040.397497:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969472 00000100:00100000:0.0:1767320040.397544:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 512 00000100:00100000:0.0:1767320040.397548:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982969472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320040.397558:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320040.397602:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982969472:12345-0@lo:38:kworker.0 Request processed in 53us (833us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320040.397609:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 512 00010000:00080000:0.0:1767320040.397648:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190a680 x1853163982968960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320056 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320040.397658:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982968960:0@lo:400:kworker.0 00000100:00080000:2.0:1767320040.397768:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320040.397773:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320040.397776:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982969472:0@lo:38:kworker.0 00010000:00010000:2.0:1767320040.581963:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320041.604665:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320042.384184:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320042.384237:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320042.384407:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659008 00000100:00100000:1.0:1767320042.384418:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659008 00000100:00100000:1.0:1767320042.384466:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659136 00000100:00100000:1.0:1767320042.384470:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659136 00000100:00100000:1.0:1767320042.384517:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1477 00000100:00100000:1.0:1767320042.384528:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915659136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320042.384545:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1951 00000100:00100000:3.0:1767320042.384554:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915659008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320042.384577:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320042.384642:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915659136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (404us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320042.384651:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1477 00000100:00080000:1.0:1767320042.384678:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320042.386285:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915659008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 1704us (2085us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320042.386302:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1951 00010000:00010000:2.0:1767320042.628691:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320042.949501:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982969984:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320042.949580:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320042.950074:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982969984 00000100:00100000:0.0:1767320042.950082:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982969984 00000100:00100000:0.0:1767320042.950138:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 994 00000100:00100000:0.0:1767320042.950145:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982969984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320042.950171:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320042.950211:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982969984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (632us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320042.950216:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 994 00000100:00100000:0.0:1767320042.950250:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982969984:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320042.950473:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320042.950591:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320042.950603:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982970112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320042.950636:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320042.950666:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970112 00000100:00100000:1.0:1767320042.950670:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970112 00000100:00100000:1.0:1767320042.950701:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995 00000100:00100000:1.0:1767320042.950706:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982970112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320042.950722:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320042.950753:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982970112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320042.950758:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995 00000100:00080000:1.0:1767320042.950772:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320042.950792:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982970112:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320043.652529:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320044.676513:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320045.508394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320045.508447:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320045.508529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320045.508533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320045.508537:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320045.508545:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320045.508552:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320045.508558:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320045.508591:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982970368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320045.508631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320045.508635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320045.508637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320045.508640:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320045.508643:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320045.508658:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320045.508674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb800 x1853163982970624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320045.508684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320045.508686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320045.508688:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320045.508697:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320045.508699:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320045.508708:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320045.508727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163982970752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320045.508770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320045.508774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320045.508776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320045.508779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320045.508781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320045.508785:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320045.508794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627480 x1853163982971136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320045.508822:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982970496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.508891:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320045.508927:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320045.508933:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982970880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.508958:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320045.508992:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970496 00000100:00100000:3.0:1767320045.509000:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970496 00000100:00100000:3.0:1767320045.509080:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1478 00000100:00100000:3.0:1767320045.509090:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982970496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.509138:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982970496:12345-0@lo:400:kworker.0 Request processed in 49us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320045.509146:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1478 00000100:00100000:3.0:1767320045.509169:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970880 00000100:00100000:3.0:1767320045.509173:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970880 00000100:00100000:3.0:1767320045.509199:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 925 00000100:00100000:3.0:1767320045.509205:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982970880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.509248:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982970880:12345-0@lo:400:kworker.0 Request processed in 43us (289us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320045.509255:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 925 00000100:00080000:3.0:1767320045.509295:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320045.509299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320045.509330:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982970880:0@lo:400:kworker.0 00010000:00080000:3.0:1767320045.509342:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0e00 x1853163982970496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320061 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320045.509351:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982970496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.509540:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982971008:0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.509587:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320045.509615:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320045.509620:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982970240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.509642:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320045.509678:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970240 00000100:00100000:0.0:1767320045.509682:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970240 00000100:00100000:0.0:1767320045.509724:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1479 00000100:00100000:0.0:1767320045.509732:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982970240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320045.509779:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982970240:12345-0@lo:400:kworker.0 Request processed in 49us (136us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320045.509786:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1479 00000100:00080000:0.0:1767320045.509802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320045.509818:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3480 x1853163982970240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320061 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320045.509828:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982970240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.510094:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971008 00000100:00100000:3.0:1767320045.510098:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971008 00000100:00100000:2.0:1767320045.510216:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982970368:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.510250:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.510300:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982970624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.510324:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.510339:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982970752:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.510361:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.510394:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982971136:0@lo:38:kworker.0 00000100:00100000:2.0:1767320045.510415:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320045.510453:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970368 00000100:00100000:2.0:1767320045.510457:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970368 00000100:00100000:2.0:1767320045.510471:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970624 00000100:00100000:2.0:1767320045.510473:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970624 00000100:00100000:2.0:1767320045.510480:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971136 00000100:00100000:2.0:1767320045.510482:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971136 00000100:00100000:2.0:1767320045.510491:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1952 00000100:00100000:2.0:1767320045.510496:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982970368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320045.510530:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320045.510565:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982970368:12345-0@lo:38:kworker.0 Request processed in 68us (314us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320045.510571:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1952 00000100:00100000:2.0:1767320045.510583:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953 00000100:00100000:2.0:1767320045.510587:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982970624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320045.510596:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320045.510619:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982970624:12345-0@lo:38:kworker.0 Request processed in 31us (295us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320045.510625:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953 00000100:00100000:2.0:1767320045.510636:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1954 00000100:00100000:2.0:1767320045.510640:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982971136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320045.510648:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320045.510669:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982971136:12345-0@lo:38:kworker.0 Request processed in 27us (253us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320045.510677:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1954 00000100:00100000:2.0:1767320045.510708:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982970752 00000100:00100000:2.0:1767320045.510713:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982970752 00000100:00100000:2.0:1767320045.510746:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 513 00000100:00100000:2.0:1767320045.510750:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982970752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320045.510759:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320045.510784:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982970752:12345-0@lo:38:kworker.0 Request processed in 32us (421us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320045.510789:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 513 00000100:00080000:2.0:1767320045.510827:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.510832:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.510836:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982970368:0@lo:38:kworker.0 00000100:00080000:2.0:1767320045.510843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.510848:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.510850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982970624:0@lo:38:kworker.0 00000100:00080000:2.0:1767320045.510857:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.510860:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.510863:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982970752:0@lo:38:kworker.0 00000100:00080000:2.0:1767320045.510869:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320045.510872:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320045.510875:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982971136:0@lo:38:kworker.0 00000100:00100000:3.0:1767320045.510976:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926 00000100:00100000:3.0:1767320045.510984:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982971008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320045.511027:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982971008:12345-0@lo:400:kworker.0 Request processed in 45us (1439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320045.511033:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926 00000100:00100000:3.0:1767320045.511072:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982971008:0@lo:400:kworker.0 00000100:00080000:0.0:1767320045.515642:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320045.700546:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320046.728125:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320047.513084:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320047.513182:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659392 00000100:00100000:0.0:1767320047.513193:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659392 00000100:00100000:0.0:1767320047.513317:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1480 00000100:00100000:0.0:1767320047.513327:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915659392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320047.513416:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915659392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (341us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320047.513424:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1480 00000100:00080000:3.0:1767320047.516748:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320047.530210:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320047.530319:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659264 00000100:00100000:2.0:1767320047.530329:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659264 00000100:00100000:2.0:1767320047.530417:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1955 00000100:00100000:2.0:1767320047.530427:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915659264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320047.530451:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320047.530531:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915659264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 104us (332us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320047.530539:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1955 00010000:00010000:1.0:1767320047.748628:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320048.068805:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982971264:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320048.068896:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320048.069008:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971264 00000100:00100000:0.0:1767320048.069016:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971264 00000100:00100000:0.0:1767320048.069085:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996 00000100:00100000:0.0:1767320048.069095:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982971264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320048.069126:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320048.069171:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982971264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320048.069178:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996 00000100:00100000:0.0:1767320048.069224:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982971264:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320048.069411:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982971392:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320048.069462:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320048.069557:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320048.069589:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971392 00000100:00100000:2.0:1767320048.069595:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971392 00000100:00100000:2.0:1767320048.069641:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 997 00000100:00100000:2.0:1767320048.069648:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982971392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320048.069666:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320048.069699:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982971392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 52us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320048.069705:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 997 00000100:00080000:1.0:1767320048.069764:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320048.069824:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982971392:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320048.772847:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320049.796467:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320050.628504:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320050.628516:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320050.628578:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320050.628582:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320050.628586:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320050.628593:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5294 00000100:00080000:0.0:1767320050.628598:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320050.628605:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320050.628652:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2a00 x1853163982971648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320050.628724:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320050.628727:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320050.628855:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982971648:0@lo:38:kworker.0 00000100:00100000:1.0:1767320050.628918:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320050.628974:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320050.628997:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982971776:0@lo:400:kworker.0 00000100:00100000:1.0:1767320050.629022:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320050.629056:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971648 00000100:00100000:1.0:1767320050.629063:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971648 00000100:00100000:1.0:1767320050.629094:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971776 00000100:00100000:1.0:1767320050.629098:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971776 00000100:00100000:1.0:1767320050.629135:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1481 00000100:00100000:1.0:1767320050.629143:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982971776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320050.629193:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982971776:12345-0@lo:400:kworker.0 Request processed in 50us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320050.629200:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1481 00000100:00100000:1.0:1767320050.629217:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1956 00000100:00100000:1.0:1767320050.629221:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982971648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320050.629239:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320050.629297:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982971648:12345-0@lo:38:kworker.0 Request processed in 47us (352us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320050.629303:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1956 00000100:00080000:1.0:1767320050.629320:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320050.629347:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320050.629352:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320050.629355:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982971648:0@lo:38:kworker.0 00010000:00080000:1.0:1767320050.629385:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322d80 x1853163982971776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320066 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320050.629399:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982971776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320050.629589:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982971520:0@lo:400:kworker.0 00000100:00100000:2.0:1767320050.629620:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320050.629770:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320050.629775:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5294 00000100:00080000:0.0:1767320050.629778:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320050.629782:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320050.629801:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321880 x1853163982971904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320050.629826:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320050.629829:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320050.629831:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320050.629834:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5294 00000100:00080000:0.0:1767320050.629837:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320050.629840:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320050.629852:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322300 x1853163982972032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320050.629892:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320050.629895:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320050.629896:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320050.629900:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5294 00000100:00080000:0.0:1767320050.629902:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320050.629905:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320050.629916:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163982972416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320050.630105:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320050.630113:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982972160:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.630137:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.630163:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982972288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.630176:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320050.630196:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972160 00000100:00100000:3.0:1767320050.630199:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972160 00000100:00100000:3.0:1767320050.630222:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972288 00000100:00100000:3.0:1767320050.630224:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972288 00000100:00100000:3.0:1767320050.630234:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927 00000100:00100000:3.0:1767320050.630239:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982972160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.630263:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982972160:12345-0@lo:400:kworker.0 Request processed in 25us (125us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320050.630268:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927 00000100:00100000:3.0:1767320050.630293:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 928 00000100:00100000:3.0:1767320050.630297:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982972288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.630315:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982972288:12345-0@lo:400:kworker.0 Request processed in 18us (138us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320050.630318:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 928 00000100:00080000:3.0:1767320050.630328:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320050.630330:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320050.630342:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982972288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.630354:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982972160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320050.630405:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982971904:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.630437:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.630467:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982972032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.630526:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.630561:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982972416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320050.630582:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320050.630619:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972032 00000100:00100000:2.0:1767320050.630623:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972032 00000100:00100000:2.0:1767320050.630642:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 514 00000100:00100000:2.0:1767320050.630647:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982972032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320050.630662:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320050.630689:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982972032:12345-0@lo:38:kworker.0 Request processed in 41us (163us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320050.630694:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 514 00000100:00100000:2.0:1767320050.630714:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971520 00000100:00100000:2.0:1767320050.630717:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971520 00000100:00100000:2.0:1767320050.630886:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982971904 00000100:00100000:2.0:1767320050.630890:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982971904 00000100:00100000:3.0:1767320050.630993:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972416 00000100:00100000:3.0:1767320050.630997:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972416 00000100:00100000:3.0:1767320050.631018:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1482 00000100:00100000:3.0:1767320050.631025:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982971520:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320050.631064:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982971520:12345-0@lo:400:kworker.0 Request processed in 41us (1444us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320050.631069:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1482 00000100:00100000:3.0:1767320050.631080:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957 00000100:00100000:3.0:1767320050.631083:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982971904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320050.631093:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320050.631115:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982971904:12345-0@lo:38:kworker.0 Request processed in 30us (687us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320050.631120:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957 00000100:00100000:3.0:1767320050.631130:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1767320050.631133:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982972416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320050.631141:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320050.631161:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982972416:12345-0@lo:38:kworker.0 Request processed in 26us (578us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320050.631166:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1958 00000100:00080000:3.0:1767320050.631181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320050.631212:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320050.631217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320050.631220:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982971904:0@lo:38:kworker.0 00000100:00080000:3.0:1767320050.631228:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320050.631231:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320050.631234:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982972032:0@lo:38:kworker.0 00000100:00080000:3.0:1767320050.631241:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320050.631243:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320050.631246:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982972416:0@lo:38:kworker.0 00010000:00080000:3.0:1767320050.631310:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b0700 x1853163982971520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320066 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320050.631320:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982971520:0@lo:400:kworker.0 00010000:00010000:1.0:1767320050.823805:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320051.844591:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320052.620440:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320052.620484:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320052.620568:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659520 00000100:00100000:0.0:1767320052.620578:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659520 00000100:00100000:0.0:1767320052.620619:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659648 00000100:00100000:0.0:1767320052.620623:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659648 00000100:00100000:0.0:1767320052.620669:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1483 00000100:00100000:0.0:1767320052.620688:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915659648:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320052.620764:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915659648:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320052.620771:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1483 00000100:00100000:0.0:1767320052.620788:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1959 00000100:00100000:0.0:1767320052.620792:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915659520:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320052.620812:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320052.620836:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915659520:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (408us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320052.620842:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1959 00000100:00080000:3.0:1767320052.621329:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320052.870253:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320053.188509:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320053.188531:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982972544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320053.188561:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320053.188575:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982972672:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320053.188601:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320053.188633:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972544 00000100:00100000:3.0:1767320053.188642:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972544 00000100:00100000:3.0:1767320053.188715:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998 00000100:00100000:3.0:1767320053.188725:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982972544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320053.188762:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320053.188809:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982972544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320053.188818:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998 00000100:00100000:2.0:1767320053.188842:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982972544:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320053.188875:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320053.188886:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320053.188921:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972672 00000100:00100000:2.0:1767320053.188925:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972672 00000100:00100000:2.0:1767320053.188964:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999 00000100:00080000:3.0:1767320053.188969:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320053.188974:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982972672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320053.188991:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320053.189034:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982972672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (147us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320053.189041:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999 00000100:00100000:3.0:1767320053.189512:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982972672:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320053.892743:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320054.916781:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320055.748762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320055.748772:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320055.748865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320055.748869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320055.748873:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320055.748882:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5299 00000100:00080000:2.0:1767320055.748887:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320055.748895:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320055.748918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4a80 x1853163982972928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320055.748948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320055.748952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320055.748953:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320055.748957:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5299 00000100:00080000:2.0:1767320055.748959:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320055.748963:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320055.748974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7800 x1853163982973184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320055.748983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320055.748986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320055.748988:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320055.748991:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5299 00000100:00080000:2.0:1767320055.748994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320055.748998:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320055.749027:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6680 x1853163982973312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320055.749095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320055.749106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320055.749108:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320055.749111:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5299 00000100:00080000:2.0:1767320055.749114:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320055.749118:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320055.749129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50a80 x1853163982973696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320055.749161:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982973056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320055.749237:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320055.749323:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320055.749330:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982973440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320055.749370:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320055.749614:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982972800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.749656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320055.749685:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320055.749691:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982973568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.749713:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320055.749741:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973440 00000100:00100000:0.0:1767320055.749748:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973440 00000100:00100000:0.0:1767320055.749808:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973568 00000100:00100000:0.0:1767320055.749812:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973568 00000100:00100000:0.0:1767320055.749836:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1767320055.749844:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982973440:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.749894:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982973440:12345-0@lo:400:kworker.0 Request processed in 51us (522us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320055.749901:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1767320055.749918:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 930 00000100:00100000:0.0:1767320055.749924:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982973568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.749969:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982973568:12345-0@lo:400:kworker.0 Request processed in 45us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320055.749975:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 930 00000100:00080000:0.0:1767320055.749991:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320055.749995:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320055.750027:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982973568:0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.750052:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982973440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320055.752122:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982972928:0@lo:38:kworker.0 00000100:00100000:3.0:1767320055.752182:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320055.752238:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982973184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320055.752261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320055.752310:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982973312:0@lo:38:kworker.0 00000100:00100000:3.0:1767320055.752345:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320055.752372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982973696:0@lo:38:kworker.0 00000100:00100000:3.0:1767320055.752394:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320055.752422:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973056 00000100:00100000:3.0:1767320055.752429:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973056 00000100:00100000:3.0:1767320055.752481:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972800 00000100:00100000:3.0:1767320055.752486:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972800 00000100:00100000:0.0:1767320055.752530:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982972928 00000100:00100000:0.0:1767320055.752535:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982972928 00000100:00100000:0.0:1767320055.752550:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973184 00000100:00100000:0.0:1767320055.752553:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973184 00000100:00100000:0.0:1767320055.752559:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973696 00000100:00100000:0.0:1767320055.752562:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973696 00000100:00100000:0.0:1767320055.752572:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1484 00000100:00100000:0.0:1767320055.752580:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982973056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.752636:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982973056:12345-0@lo:400:kworker.0 Request processed in 57us (3410us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320055.752643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1484 00000100:00100000:0.0:1767320055.752661:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1960 00000100:00100000:0.0:1767320055.752664:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982972928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320055.752692:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320055.752723:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982972928:12345-0@lo:38:kworker.0 Request processed in 56us (540us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320055.752729:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1960 00000100:00100000:0.0:1767320055.752742:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1961 00000100:00100000:0.0:1767320055.752745:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982973184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320055.752760:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320055.752760:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1485 00000100:00100000:3.0:1767320055.752766:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982972800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320055.752781:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982973184:12345-0@lo:38:kworker.0 Request processed in 35us (520us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320055.752787:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1961 00000100:00100000:0.0:1767320055.752797:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1962 00000100:00100000:0.0:1767320055.752801:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982973696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320055.752809:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320055.752831:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982973696:12345-0@lo:38:kworker.0 Request processed in 28us (436us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320055.752836:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1962 00010000:00080000:0.0:1767320055.752870:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6a00 x1853163982973056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320071 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320055.752883:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982973056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320055.752992:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982972800:12345-0@lo:400:kworker.0 Request processed in 225us (3335us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320055.752999:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1485 00000100:00080000:3.0:1767320055.753016:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320055.753035:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7100 x1853163982972800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320071 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320055.753045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982972800:0@lo:400:kworker.0 00000100:00080000:3.0:1767320055.753066:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320055.753071:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320055.753074:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982972928:0@lo:38:kworker.0 00000100:00080000:3.0:1767320055.753082:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320055.753086:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320055.753088:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982973184:0@lo:38:kworker.0 00000100:00080000:3.0:1767320055.753095:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320055.753098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320055.753100:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982973696:0@lo:38:kworker.0 00000100:00100000:0.0:1767320055.753179:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973312 00000100:00100000:0.0:1767320055.753184:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973312 00000100:00100000:0.0:1767320055.753209:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 515 00000100:00100000:0.0:1767320055.753213:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982973312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320055.753222:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320055.753353:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982973312:12345-0@lo:38:kworker.0 Request processed in 138us (1007us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320055.753360:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 515 00000100:00080000:3.0:1767320055.753833:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320055.753838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320055.753841:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982973312:0@lo:38:kworker.0 00010000:00010000:2.0:1767320055.943170:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320056.965187:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320057.745008:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320057.745060:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320057.745450:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659904 00000100:00100000:2.0:1767320057.745462:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659904 00000100:00100000:3.0:1767320057.745493:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915659776 00000100:00100000:3.0:1767320057.745497:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915659776 00000100:00100000:3.0:1767320057.745529:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1486 00000100:00100000:3.0:1767320057.745539:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915659904:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320057.745616:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915659904:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (619us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320057.745624:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1486 00000100:00100000:3.0:1767320057.745654:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1963 00000100:00100000:3.0:1767320057.745660:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915659776:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320057.745678:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320057.745702:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915659776:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (642us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320057.745708:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1963 00000100:00080000:0.0:1767320057.745756:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320057.988567:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320058.308885:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320058.308924:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982973824:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320058.309302:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320058.309592:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320058.309601:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982973952:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320058.309624:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320058.310357:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973824 00000100:00100000:2.0:1767320058.310366:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973824 00000100:00100000:2.0:1767320058.310445:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982973952 00000100:00100000:2.0:1767320058.310449:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982973952 00000100:00100000:2.0:1767320058.310471:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000 00000100:00100000:2.0:1767320058.310481:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982973824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320058.310532:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320058.310584:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982973824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (1283us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320058.310589:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000 00000100:00100000:2.0:1767320058.310602:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001 00000100:00100000:2.0:1767320058.310606:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982973952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320058.310618:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320058.310635:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982973952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 29us (1010us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320058.310640:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001 00000100:00100000:1.0:1767320058.310846:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982973824:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320058.311376:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320058.311381:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320058.311403:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982973952:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320059.025347:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320060.037609:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320060.868431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320060.868442:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320060.868515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320060.868520:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320060.868524:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320060.868532:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5304 00000100:00080000:2.0:1767320060.868537:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320060.868543:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320060.868567:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52300 x1853163982974208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320060.868615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320060.868618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320060.868620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320060.868624:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5304 00000100:00080000:2.0:1767320060.868627:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320060.868631:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320060.868642:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51500 x1853163982974464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320060.868650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320060.868653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320060.868655:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320060.868658:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5304 00000100:00080000:2.0:1767320060.868660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320060.868664:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320060.868674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50700 x1853163982974592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320060.868696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320060.868700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320060.868701:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320060.868705:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5304 00000100:00080000:2.0:1767320060.868707:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320060.868710:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320060.868719:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51f80 x1853163982974976/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320060.868794:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982974720:0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.868866:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320060.868944:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320060.868950:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982974336:0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.868977:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320060.869007:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982974080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.869030:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320060.869055:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320060.869061:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982974848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.869082:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320060.869115:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974720 00000100:00100000:3.0:1767320060.869122:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974720 00000100:00100000:3.0:1767320060.869171:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974848 00000100:00100000:3.0:1767320060.869174:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974848 00000100:00100000:3.0:1767320060.869197:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 931 00000100:00100000:3.0:1767320060.869206:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982974720:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.869290:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982974720:12345-0@lo:400:kworker.0 Request processed in 63us (402us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320060.869300:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 931 00000100:00100000:3.0:1767320060.869328:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 932 00000100:00100000:3.0:1767320060.869335:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982974848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.869366:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982974848:12345-0@lo:400:kworker.0 Request processed in 33us (283us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320060.869372:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 932 00000100:00080000:3.0:1767320060.869389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320060.869393:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320060.869426:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982974848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320060.869450:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982974720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320060.869615:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974336 00000100:00100000:2.0:1767320060.869620:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974336 00000100:00100000:2.0:1767320060.869656:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974080 00000100:00100000:2.0:1767320060.869659:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974080 00000100:00100000:2.0:1767320060.869678:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1487 00000100:00100000:2.0:1767320060.869684:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982974336:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320060.869718:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982974336:12345-0@lo:400:kworker.0 Request processed in 35us (741us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320060.869724:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1487 00000100:00100000:2.0:1767320060.869735:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1488 00000100:00100000:2.0:1767320060.869741:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982974080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320060.869770:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982974080:12345-0@lo:400:kworker.0 Request processed in 30us (740us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320060.869775:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1488 00000100:00100000:0.0:1767320060.869921:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982974208:0@lo:38:kworker.0 00000100:00100000:0.0:1767320060.869954:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320060.869986:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982974464:0@lo:38:kworker.0 00000100:00100000:0.0:1767320060.870007:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320060.870028:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982974592:0@lo:38:kworker.0 00000100:00100000:0.0:1767320060.870052:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320060.870073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982974976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320060.870152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320060.870183:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974592 00000100:00100000:0.0:1767320060.870187:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974592 00000100:00100000:0.0:1767320060.870207:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 516 00000100:00100000:0.0:1767320060.870211:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982974592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320060.870228:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320060.870268:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982974592:12345-0@lo:38:kworker.0 Request processed in 55us (215us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320060.870297:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 516 00000100:00100000:0.0:1767320060.870313:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974208 00000100:00100000:0.0:1767320060.870328:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974208 00000100:00100000:0.0:1767320060.870340:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974464 00000100:00100000:0.0:1767320060.870342:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974464 00000100:00100000:0.0:1767320060.870347:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982974976 00000100:00100000:0.0:1767320060.870349:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982974976 00000100:00100000:0.0:1767320060.870356:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1964 00000100:00100000:0.0:1767320060.870360:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974208:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320060.870368:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320060.870391:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974208:12345-0@lo:38:kworker.0 Request processed in 30us (437us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320060.870400:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1964 00000100:00100000:0.0:1767320060.870409:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1965 00000100:00100000:0.0:1767320060.870412:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974464:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320060.870419:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320060.870437:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974464:12345-0@lo:38:kworker.0 Request processed in 24us (430us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320060.870441:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1965 00000100:00100000:0.0:1767320060.870450:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1966 00000100:00100000:0.0:1767320060.870453:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974976:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320060.870460:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320060.870475:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982974976:12345-0@lo:38:kworker.0 Request processed in 21us (323us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320060.870480:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1966 00000100:00080000:0.0:1767320060.870509:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320060.870514:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320060.870518:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982974208:0@lo:38:kworker.0 00000100:00080000:0.0:1767320060.870526:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320060.870529:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320060.870532:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982974464:0@lo:38:kworker.0 00000100:00080000:0.0:1767320060.870539:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320060.870541:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320060.870543:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982974592:0@lo:38:kworker.0 00000100:00080000:0.0:1767320060.870550:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320060.870553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320060.870556:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982974976:0@lo:38:kworker.0 00000100:00080000:3.0:1767320060.870866:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320060.870888:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f51880 x1853163982974080/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320076 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320060.870967:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982974080:0@lo:400:kworker.0 00010000:00080000:3.0:1767320060.870993:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f53480 x1853163982974336/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320076 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320060.874108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982974336:0@lo:400:kworker.0 00010000:00010000:2.0:1767320061.060512:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320061.836102:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320061.836383:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660288 00000100:00100000:3.0:1767320061.836394:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660288 00000100:00080000:0.0:1767320061.836665:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767320061.836699:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915660032 offset 0 length 520: 4 00000100:00100000:1.0:1767320061.836725:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320061.843185:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660160 00000100:00100000:2.0:1767320061.843201:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660160 00000100:00100000:2.0:1767320061.843289:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1489 00000100:00100000:2.0:1767320061.843307:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915660288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320061.843431:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915660288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (7337us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320061.843439:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1489 00000100:00100000:2.0:1767320061.843462:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1967 00000100:00100000:2.0:1767320061.843466:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915660160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320061.843496:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320061.843524:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915660160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (6799us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320061.843530:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1967 00010000:00010000:2.0:1767320062.084816:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320063.108503:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320063.428751:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320063.428774:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982975104:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320063.428851:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320063.428954:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320063.428963:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982975232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320063.428989:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320063.429077:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975104 00000100:00100000:2.0:1767320063.429081:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975232 00000100:00100000:3.0:1767320063.429084:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975104 00000100:00100000:2.0:1767320063.429088:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975232 00000100:00100000:2.0:1767320063.429151:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1002 00000100:00080000:0.0:1767320063.429161:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320063.429162:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982975232:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320063.429168:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767320063.429229:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320063.429310:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982975232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 150us (318us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320063.429319:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1002 00000100:00100000:2.0:1767320063.429340:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1003 00000100:00100000:0.0:1767320063.429349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982975232:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320063.429350:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982975104:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320063.429373:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320063.429414:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982975104:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (564us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320063.429420:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1003 00000100:00100000:0.0:1767320063.429475:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982975104:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320064.132603:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320065.156497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320065.988656:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320040/real 1767320040] req@ffff934631909f80 x1853163982968832/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320065 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320065.988742:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631909f80 x1853163982968832/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320065 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320065.988763:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320065.989007:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320065.989012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982968832:0@lo:250:kworker.0 00000100:00080000:3.0:1767320065.989136:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.989149:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320065.989152:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.989157:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5288 00000100:00080000:3.0:1767320065.989161:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.989167:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320065.989201:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163982975360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.989257:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.989260:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.989262:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.989266:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.989286:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.989290:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320065.989307:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625880 x1853163982975616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.989329:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.989332:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.989334:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.989337:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.989340:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.989344:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320065.989355:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0700 x1853163982975872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.989364:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.989367:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.989369:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.989372:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.989385:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.989389:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320065.989403:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982976000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320065.989425:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320065.989428:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320065.989429:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320065.989433:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5309 00000100:00080000:3.0:1767320065.989436:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320065.989439:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320065.989449:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163982976384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320065.989476:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982976256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.989537:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320065.989594:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320065.989600:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982975488:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.989628:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320065.990068:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982975744:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.990116:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320065.990140:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320065.990156:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982976128:0@lo:400:kworker.0 00000100:00100000:3.0:1767320065.990179:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320065.990210:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976256 00000100:00100000:3.0:1767320065.990217:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976256 00000100:00100000:3.0:1767320065.990266:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976128 00000100:00100000:3.0:1767320065.990290:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976128 00000100:00100000:3.0:1767320065.990316:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933 00000100:00100000:3.0:1767320065.990325:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982976256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.990479:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982975360:0@lo:250:kworker.0 00000100:00080000:2.0:1767320065.990487:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320065.990493:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000400:00000100:1.0:1767320065.990507:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982975360 offset 0 length 520: 4 00000100:00100000:2.0:1767320065.990523:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 934 00000100:00100000:1.0:1767320065.990528:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982975616:0@lo:38:kworker.0 00000100:00100000:2.0:1767320065.990536:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982976128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.990553:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320065.990573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982975872:0@lo:38:kworker.0 00000100:00100000:1.0:1767320065.990605:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320065.990606:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982976128:12345-0@lo:400:kworker.0 Request processed in 71us (421us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320065.990615:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 934 00000100:00100000:1.0:1767320065.990627:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982976000:0@lo:38:kworker.0 00000100:00100000:2.0:1767320065.990644:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975488 00000100:00100000:2.0:1767320065.990650:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975488 00000100:00100000:1.0:1767320065.990664:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320065.990682:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975744 00000100:00100000:2.0:1767320065.990686:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975744 00000100:00100000:1.0:1767320065.990698:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982976384:0@lo:38:kworker.0 00000100:00100000:2.0:1767320065.990701:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975616 00000100:00100000:2.0:1767320065.990706:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975616 00000100:00100000:2.0:1767320065.990719:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982975872 00000100:00100000:2.0:1767320065.990722:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982975872 00000100:00100000:1.0:1767320065.990727:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320065.990736:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976384 00000100:00100000:2.0:1767320065.990740:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976384 00000100:00100000:2.0:1767320065.990754:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1490 00000100:00100000:1.0:1767320065.990759:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976000 00000100:00100000:1.0:1767320065.990764:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976000 00000100:00100000:2.0:1767320065.990764:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982975488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.990935:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 517 00000100:00100000:1.0:1767320065.990940:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982976000:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320065.990958:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320065.990993:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982976000:12345-0@lo:38:kworker.0 Request processed in 51us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320065.990999:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 517 00000100:00080000:1.0:1767320065.991037:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.991041:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.991044:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982976000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320065.991080:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1491 00000100:00100000:1.0:1767320065.991088:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982975744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320065.991119:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982975744:12345-0@lo:400:kworker.0 Request processed in 32us (1002us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320065.991128:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1491 00000100:00100000:1.0:1767320065.991154:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968 00000100:00100000:1.0:1767320065.991159:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982975616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320065.991170:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320065.991180:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982975488:12345-0@lo:400:kworker.0 Request processed in 418us (1550us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320065.991188:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1490 00000100:00100000:1.0:1767320065.991207:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982975616:12345-0@lo:38:kworker.0 Request processed in 45us (652us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320065.991208:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969 00000100:00100000:2.0:1767320065.991217:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982975872:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320065.991220:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968 00010000:00020000:2.0:1767320065.991234:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320065.991238:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1970 00000100:00100000:1.0:1767320065.991243:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982976384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320065.991253:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320065.991263:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982975872:12345-0@lo:38:kworker.0 Request processed in 45us (661us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320065.991295:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969 00000100:00100000:1.0:1767320065.991318:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982976384:12345-0@lo:38:kworker.0 Request processed in 72us (590us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320065.991326:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1970 00010000:00080000:2.0:1767320065.991352:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626d80 x1853163982975488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320081 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320065.991357:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320065.991368:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982975488:0@lo:400:kworker.0 00000100:00100000:2.0:1767320065.991383:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982976256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320065.991419:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982976128:0@lo:400:kworker.0 00010000:00080000:2.0:1767320065.991428:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c627480 x1853163982975744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320081 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320065.991448:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982975744:0@lo:400:kworker.0 00000100:00080000:1.0:1767320065.991476:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320065.991482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320065.991485:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982975616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320065.991556:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982976256:12345-0@lo:400:kworker.0 Request processed in 1231us (2018us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320065.991564:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933 00000100:00080000:2.0:1767320065.992759:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320065.992766:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320065.992919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982975872:0@lo:38:kworker.0 00000100:00080000:2.0:1767320065.992935:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320065.992938:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320065.992941:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982976384:0@lo:38:kworker.0 00010000:00010000:1.0:1767320066.183995:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320066.957880:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320066.957995:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320066.958091:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660544 00000100:00100000:0.0:1767320066.958103:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660544 00000100:00100000:0.0:1767320066.958187:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660416 00000100:00100000:0.0:1767320066.958191:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660416 00000100:00100000:0.0:1767320066.958209:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1492 00000100:00100000:0.0:1767320066.958219:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915660544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320066.958331:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915660544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 114us (462us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320066.958339:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1492 00000100:00100000:0.0:1767320066.958359:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1971 00000100:00100000:0.0:1767320066.958364:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915660416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320066.958386:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320066.958410:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915660416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320066.958416:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1971 00000100:00080000:1.0:1767320066.958573:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320067.204541:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320068.228583:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320068.548563:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320068.548587:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982976512:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320068.548671:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320068.548885:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976512 00000100:00100000:3.0:1767320068.548893:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976512 00000100:00080000:1.0:1767320068.548928:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320068.548976:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004 00000100:00100000:3.0:1767320068.548987:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982976512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320068.549021:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320068.549076:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982976512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (405us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320068.549108:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004 00000100:00100000:1.0:1767320068.549121:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982976512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320068.549507:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982976640:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320068.549543:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320068.549591:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976640 00000100:00100000:0.0:1767320068.549596:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976640 00000100:00100000:0.0:1767320068.549643:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005 00000100:00100000:0.0:1767320068.549651:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982976640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320068.549669:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320068.549702:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982976640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (158us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320068.549707:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005 00000100:00080000:0.0:1767320068.549724:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320068.549756:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982976640:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320069.252552:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320070.276928:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320071.108807:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320071.108818:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320071.108883:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320071.108888:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320071.108892:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320071.108899:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5314 00000100:00080000:0.0:1767320071.108903:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320071.108910:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320071.108933:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323b80 x1853163982976896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320071.109012:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320071.109016:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320071.109017:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320071.109021:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5314 00000100:00080000:0.0:1767320071.109023:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320071.109027:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320071.109037:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322680 x1853163982977152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320071.109045:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320071.109058:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320071.109060:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320071.109063:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5314 00000100:00080000:0.0:1767320071.109065:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320071.109069:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320071.109093:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321f80 x1853163982977280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320071.109115:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320071.109118:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320071.109120:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320071.109124:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5314 00000100:00080000:0.0:1767320071.109126:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320071.109129:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320071.110310:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982977536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110390:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.110432:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320071.110439:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982976768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110467:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.110509:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976768 00000100:00100000:2.0:1767320071.110518:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976768 00000100:00100000:2.0:1767320071.110575:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1493 00000100:00100000:2.0:1767320071.110597:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982976768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110648:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982976768:12345-0@lo:400:kworker.0 Request processed in 66us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320071.110656:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1493 00010000:00080000:2.0:1767320071.110693:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320a80 x1853163982976768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320087 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320071.110705:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982976768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110749:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982977408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110772:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.110791:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320071.110797:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982977024:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110818:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.110841:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977536 00000100:00100000:2.0:1767320071.110846:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977536 00000100:00100000:2.0:1767320071.110871:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977408 00000100:00100000:2.0:1767320071.110875:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977408 00000100:00100000:2.0:1767320071.110894:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935 00000100:00100000:2.0:1767320071.110900:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982977536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110934:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982977536:12345-0@lo:400:kworker.0 Request processed in 34us (543us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320071.110940:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935 00000100:00100000:2.0:1767320071.110952:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936 00000100:00100000:2.0:1767320071.110959:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982977408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.110992:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982977408:12345-0@lo:400:kworker.0 Request processed in 34us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320071.110998:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936 00000100:00100000:2.0:1767320071.111024:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982976896:0@lo:38:kworker.0 00000100:00100000:2.0:1767320071.111060:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.111080:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982977152:0@lo:38:kworker.0 00000100:00100000:2.0:1767320071.111102:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.111119:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982977280:0@lo:38:kworker.0 00000100:00100000:2.0:1767320071.111141:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320071.111176:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982977408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.111371:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977024 00000100:00100000:2.0:1767320071.111375:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977024 00000100:00100000:2.0:1767320071.111399:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982976896 00000100:00100000:2.0:1767320071.111403:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982976896 00000100:00100000:2.0:1767320071.111410:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977152 00000100:00100000:2.0:1767320071.111412:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977152 00000100:00100000:2.0:1767320071.111421:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1494 00000100:00100000:2.0:1767320071.111427:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982977024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320071.111463:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982977024:12345-0@lo:400:kworker.0 Request processed in 36us (644us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320071.111469:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1494 00000100:00100000:2.0:1767320071.111482:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1972 00000100:00100000:2.0:1767320071.111486:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982976896:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320071.111542:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320071.111574:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982976896:12345-0@lo:38:kworker.0 Request processed in 86us (524us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320071.111693:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1972 00000100:00100000:0.0:1767320071.112260:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321880 x1853163982977664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320071.112306:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320071.112312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320071.112315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320071.117199:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1973 00000100:00100000:1.0:1767320071.117214:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982977152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320071.117237:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320071.117336:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982977152:12345-0@lo:38:kworker.0 Request processed in 124us (6231us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320071.117344:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1973 00000100:00100000:1.0:1767320071.117390:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977280 00000100:00100000:1.0:1767320071.117398:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977280 00000100:00100000:1.0:1767320071.117428:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 518 00000100:00100000:1.0:1767320071.117432:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982977280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320071.117442:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320071.117470:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982977280:12345-0@lo:38:kworker.0 Request processed in 37us (6328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320071.117480:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 518 00000100:00100000:1.0:1767320071.117654:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982977536:0@lo:400:kworker.0 00010000:00080000:1.0:1767320071.117698:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321180 x1853163982977024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320087 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320071.117710:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982977024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320071.117737:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982977664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320071.117774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320071.117805:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320071.117810:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320071.117813:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982976896:0@lo:38:kworker.0 00000100:00080000:1.0:1767320071.117822:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320071.117825:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320071.117829:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982977152:0@lo:38:kworker.0 00000100:00080000:1.0:1767320071.117836:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320071.117840:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320071.117843:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982977280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320071.117869:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977664 00000100:00100000:1.0:1767320071.117873:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977664 00000100:00100000:1.0:1767320071.117892:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1974 00000100:00100000:1.0:1767320071.117897:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982977664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320071.117907:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320071.117952:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982977664:12345-0@lo:38:kworker.0 Request processed in 54us (178us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320071.117959:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1974 00000100:00080000:1.0:1767320071.117983:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320071.117987:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320071.117990:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982977664:0@lo:38:kworker.0 00010000:00010000:2.0:1767320071.302382:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320072.086794:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320072.086850:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320072.086954:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660800 00000100:00100000:3.0:1767320072.086966:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660800 00000100:00100000:3.0:1767320072.087078:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660672 00000100:00100000:3.0:1767320072.087081:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660672 00000100:00100000:3.0:1767320072.087101:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1495 00000100:00100000:3.0:1767320072.087111:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915660800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320072.087205:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915660800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (424us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320072.087212:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1495 00000100:00100000:3.0:1767320072.087245:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1975 00000100:00100000:3.0:1767320072.087249:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915660672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320072.087317:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320072.087347:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915660672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (496us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320072.087353:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1975 00000100:00080000:1.0:1767320072.088027:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320072.324685:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320073.349770:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320073.668955:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982977792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320073.669042:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320073.669237:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977792 00000100:00100000:1.0:1767320073.669245:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977792 00000100:00100000:1.0:1767320073.669332:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1006 00000100:00100000:1.0:1767320073.669343:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982977792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320073.669374:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767320073.669435:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320073.669466:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320073.669479:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982977920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320073.673639:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982977792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4286us (4585us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320073.673655:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1006 00000100:00100000:2.0:1767320073.678937:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320073.679090:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982977920 00000100:00100000:1.0:1767320073.679100:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982977920 00000100:00100000:1.0:1767320073.679179:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007 00000100:00100000:1.0:1767320073.679190:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982977920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320073.679224:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320073.679300:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982977920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 111us (369us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320073.679308:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007 00000100:00080000:2.0:1767320073.679499:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320073.679541:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982977920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320073.679719:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982977792:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320074.372579:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320075.396634:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320076.228820:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320076.228832:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320076.229004:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982978048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320076.229129:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320076.229226:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978048 00000100:00100000:2.0:1767320076.229235:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978048 00000100:00080000:3.0:1767320076.229359:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320076.229436:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1496 00000100:00100000:2.0:1767320076.229446:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982978048:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320076.229944:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320076.229963:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320076.229967:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320076.229973:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5319 00000100:00080000:1.0:1767320076.229977:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320076.229983:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320076.230025:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3480 x1853163982978176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320076.230073:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320076.230077:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320076.230079:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320076.230082:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5319 00000100:00080000:1.0:1767320076.230085:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320076.230089:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320076.230158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982978176:0@lo:38:kworker.0 00000100:00100000:3.0:1767320076.230210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320076.230243:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978176 00000100:00100000:3.0:1767320076.230248:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978176 00000100:00100000:3.0:1767320076.230290:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1976 00000100:00100000:3.0:1767320076.230296:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982978176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320076.230313:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320076.230351:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982978176:12345-0@lo:38:kworker.0 Request processed in 54us (140us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320076.230356:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982978432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320076.230362:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1976 00000100:00080000:1.0:1767320076.230375:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320076.230378:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320076.230381:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320076.230385:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5319 00000100:00080000:1.0:1767320076.230387:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320076.230397:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982978432:0@lo:38:kworker.0 00000100:00080000:1.0:1767320076.230407:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320076.230430:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320076.230459:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965180 x1853163982978560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320076.230460:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320076.230468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320076.230474:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982978176:0@lo:38:kworker.0 00000100:00080000:1.0:1767320076.230507:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767320076.230509:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982978560:0@lo:38:kworker.0 00000100:00080000:1.0:1767320076.230514:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320076.230517:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320076.230522:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5319 00000100:00080000:1.0:1767320076.230525:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320076.230530:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320076.230543:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966300 x1853163982978944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320076.230558:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320076.230572:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982978304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320076.230601:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982978944:0@lo:38:kworker.0 00000100:00100000:1.0:1767320076.230612:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320076.230633:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320076.230639:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320076.230646:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982978688:0@lo:400:kworker.0 00000100:00100000:1.0:1767320076.230673:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320076.230681:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978432 00000100:00100000:3.0:1767320076.230686:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978432 00000100:00100000:3.0:1767320076.230708:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978304 00000100:00100000:3.0:1767320076.230711:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978304 00000100:00100000:1.0:1767320076.230721:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978560 00000100:00100000:1.0:1767320076.230728:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978560 00000100:00100000:3.0:1767320076.230738:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978944 00000100:00100000:3.0:1767320076.230742:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978944 00000100:00100000:3.0:1767320076.230758:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1497 00000100:00100000:3.0:1767320076.230763:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982978304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320076.230779:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 519 00000100:00100000:1.0:1767320076.230785:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982978560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320076.230797:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320076.230808:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982978304:12345-0@lo:400:kworker.0 Request processed in 43us (196us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320076.230815:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1497 00000100:00100000:1.0:1767320076.230836:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982978560:12345-0@lo:38:kworker.0 Request processed in 49us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320076.230843:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 519 00000100:00100000:1.0:1767320076.230879:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977 00000100:00100000:1.0:1767320076.230884:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982978432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320076.230893:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320076.230921:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982978432:12345-0@lo:38:kworker.0 Request processed in 35us (491us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320076.230930:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977 00000100:00100000:1.0:1767320076.230942:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1978 00000100:00100000:1.0:1767320076.230946:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982978944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320076.230970:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320076.230996:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982978944:12345-0@lo:38:kworker.0 Request processed in 49us (364us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320076.231001:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978688 00000100:00100000:1.0:1767320076.231004:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1978 00000100:00100000:3.0:1767320076.231007:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978688 00000100:00100000:1.0:1767320076.231032:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982978816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320076.231044:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 937 00000100:00100000:3.0:1767320076.231052:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982978688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320076.231060:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320076.231095:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982978688:12345-0@lo:400:kworker.0 Request processed in 44us (420us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320076.231100:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 937 00000100:00100000:3.0:1767320076.231109:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982978816 00000100:00100000:3.0:1767320076.231111:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982978816 00000100:00100000:3.0:1767320076.231128:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 938 00000100:00100000:3.0:1767320076.231132:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982978816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320076.231157:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982978816:12345-0@lo:400:kworker.0 Request processed in 24us (99us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320076.231162:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 938 00000100:00080000:3.0:1767320076.231181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320076.231184:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320076.231186:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320076.231203:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982978688:0@lo:400:kworker.0 00010000:00080000:3.0:1767320076.231213:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0e00 x1853163982978304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320092 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320076.231221:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982978304:0@lo:400:kworker.0 00000100:00080000:3.0:1767320076.231244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320076.231247:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320076.231250:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982978944:0@lo:38:kworker.0 00000100:00080000:3.0:1767320076.231256:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320076.231259:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320076.231261:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982978560:0@lo:38:kworker.0 00000100:00080000:3.0:1767320076.231289:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320076.231292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320076.231295:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982978432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320076.231430:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982978816:0@lo:400:kworker.0 00000100:00100000:2.0:1767320076.237864:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982978048:12345-0@lo:400:kworker.0 Request processed in 8399us (8714us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320076.237882:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1496 00010000:00080000:0.0:1767320076.237983:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190aa00 x1853163982978048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320092 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320076.238000:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982978048:0@lo:400:kworker.0 00010000:00010000:2.0:1767320076.420573:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320077.198976:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320077.199025:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320077.199090:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661056 00000100:00100000:2.0:1767320077.199099:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661056 00000100:00100000:2.0:1767320077.199157:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915660928 00000100:00100000:2.0:1767320077.199159:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915660928 00000100:00100000:2.0:1767320077.199174:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1498 00000100:00100000:2.0:1767320077.199181:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915661056:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320077.199221:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1979 00000100:00100000:1.0:1767320077.199229:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915660928:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767320077.199240:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915661056:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 58us (276us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320077.199245:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1498 00010000:00020000:1.0:1767320077.199251:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320077.199311:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915660928:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 81us (285us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320077.199319:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1979 00000100:00080000:0.0:1767320077.199446:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320077.445361:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320078.469077:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320078.788804:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320078.788822:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982979072:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320078.788890:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320078.789059:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979072 00000100:00100000:3.0:1767320078.789068:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979072 00000100:00080000:0.0:1767320078.789115:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320078.789166:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008 00000100:00100000:3.0:1767320078.789176:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982979072:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320078.789206:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320078.789263:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982979072:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (374us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320078.789291:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008 00000100:00100000:0.0:1767320078.789334:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982979072:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320078.789362:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320078.789370:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982979200:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320078.789392:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320078.790243:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979200 00000100:00100000:1.0:1767320078.790252:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979200 00000100:00100000:0.0:1767320078.790360:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1009 00000100:00100000:0.0:1767320078.790372:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982979200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320078.790396:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767320078.790420:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320078.790512:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982979200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 144us (1118us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320078.790518:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1009 00000100:00100000:2.0:1767320078.793789:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982979200:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320079.492467:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320080.516610:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320081.348382:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320081.348393:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320081.348463:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320081.348468:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320081.348471:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320081.348479:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5324 00000100:00080000:1.0:1767320081.348484:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320081.348490:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320081.348511:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964380 x1853163982979456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320081.348541:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320081.348559:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982979328:0@lo:400:kworker.0 00000100:00080000:1.0:1767320081.348559:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320081.348567:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320081.348569:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320081.348572:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5324 00000100:00080000:1.0:1767320081.348574:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320081.348577:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320081.348591:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967800 x1853163982979712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320081.348599:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320081.348602:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320081.348604:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320081.348607:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5324 00000100:00080000:1.0:1767320081.348610:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320081.348613:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320081.348627:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965c00 x1853163982979840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320081.348650:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320081.348652:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320081.348657:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320081.348658:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320081.348662:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5324 00000100:00080000:1.0:1767320081.348664:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320081.348667:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320081.348677:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163982980224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320081.348699:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982979456:0@lo:38:kworker.0 00000100:00100000:1.0:1767320081.348718:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982979584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.348727:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982980096:0@lo:400:kworker.0 00000100:00100000:3.0:1767320081.348747:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320081.348871:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979328 00000100:00100000:2.0:1767320081.348881:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979328 00000100:00100000:2.0:1767320081.348934:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979456 00000100:00100000:2.0:1767320081.348937:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979456 00000100:00100000:2.0:1767320081.348957:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1499 00000100:00100000:2.0:1767320081.348966:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982979328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.349103:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320081.349194:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1980 00000100:00100000:0.0:1767320081.349200:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d5850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982979456:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320081.349220:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320081.349256:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d5850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982979456:12345-0@lo:38:kworker.0 Request processed in 54us (509us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320081.349263:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1980 00000100:00100000:0.0:1767320081.349330:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980096 00000100:00100000:0.0:1767320081.349335:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980096 00000100:00100000:0.0:1767320081.349369:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939 00000100:00100000:0.0:1767320081.349377:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982980096:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.349416:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982980096:12345-0@lo:400:kworker.0 Request processed in 41us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320081.349423:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939 00000100:00100000:0.0:1767320081.349461:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982980096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.349501:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982979968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.349528:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320081.349562:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979968 00000100:00100000:0.0:1767320081.349566:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979968 00000100:00100000:0.0:1767320081.349592:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940 00000100:00100000:0.0:1767320081.349600:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982979968:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320081.349648:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982979968:12345-0@lo:400:kworker.0 Request processed in 48us (118us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320081.349654:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940 00000100:00100000:0.0:1767320081.349680:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982979968:0@lo:400:kworker.0 00000100:00080000:0.0:1767320081.349723:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320081.349727:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320081.349730:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320081.352717:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320081.352730:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320081.352736:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982979456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320081.352741:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982979328:12345-0@lo:400:kworker.0 Request processed in 3771us (4088us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320081.352750:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982979712:0@lo:38:kworker.0 00000100:00100000:2.0:1767320081.352757:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1499 00000100:00100000:1.0:1767320081.353975:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320081.354025:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979584 00000100:00100000:3.0:1767320081.354042:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979584 00000100:00080000:1.0:1767320081.354112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320081.354118:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1500 00000100:00100000:3.0:1767320081.354130:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982979584:12345-0@lo:400:kworker.0 00010000:00080000:0.0:1767320081.354135:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965f80 x1853163982979328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320097 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320081.354138:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320081.354150:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982979328:0@lo:400:kworker.0 00000100:00100000:1.0:1767320081.354172:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982979840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320081.354174:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979712 00000100:00100000:0.0:1767320081.354177:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979712 00000100:00100000:0.0:1767320081.354201:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981 00000100:00100000:0.0:1767320081.354207:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982979712:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320081.354214:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320081.354219:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982979584:12345-0@lo:400:kworker.0 Request processed in 91us (246us total) trans 0 rc 0/0 00010000:00020000:0.0:1767320081.354226:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320081.354231:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1500 00000100:00100000:1.0:1767320081.354249:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982980224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320081.354263:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982979712:12345-0@lo:38:kworker.0 Request processed in 54us (124us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320081.354289:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981 00000100:00100000:3.0:1767320081.354303:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982979840 00000100:00100000:1.0:1767320081.354309:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320081.354310:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982979840 00010000:00080000:0.0:1767320081.354334:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964000 x1853163982979584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320097 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320081.354345:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982979584:0@lo:400:kworker.0 00000100:00100000:3.0:1767320081.354351:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 520 00000100:00100000:3.0:1767320081.354358:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638756050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982979840:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320081.354369:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320081.354373:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980224 00000100:00100000:0.0:1767320081.354378:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980224 00000100:00100000:0.0:1767320081.354400:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1982 00000100:00100000:0.0:1767320081.354405:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982980224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320081.354415:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320081.354441:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982980224:12345-0@lo:38:kworker.0 Request processed in 35us (134us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320081.354442:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320081.354446:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320081.354447:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1982 00000100:00100000:1.0:1767320081.354449:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982980224:0@lo:38:kworker.0 00000100:00080000:1.0:1767320081.354468:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320081.354472:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320081.354475:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982979712:0@lo:38:kworker.0 00000100:00100000:3.0:1767320081.355658:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638756050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982979840:12345-0@lo:38:kworker.0 Request processed in 1293us (1436us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320081.355670:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 520 00000100:00080000:1.0:1767320081.355692:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320081.355697:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320081.355701:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982979840:0@lo:38:kworker.0 00010000:00010000:1.0:1767320081.540502:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320082.319213:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320082.319331:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320082.319542:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661312 00000100:00100000:2.0:1767320082.319553:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661312 00000100:00100000:2.0:1767320082.319633:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661184 00000100:00080000:0.0:1767320082.319637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320082.319638:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661184 00000100:00100000:2.0:1767320082.319666:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1501 00000100:00100000:0.0:1767320082.319675:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1983 00000100:00100000:2.0:1767320082.319680:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915661312:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320082.319685:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915661184:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320082.319705:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320082.319751:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915661312:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (548us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320082.319757:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1501 00000100:00100000:0.0:1767320082.319821:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915661184:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 136us (489us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320082.319849:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1983 00010000:00010000:2.0:1767320082.564572:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320083.590167:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320083.911405:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320083.911440:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982980480:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320083.911514:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320083.911905:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980480 00000100:00100000:2.0:1767320083.911914:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980480 00000100:00080000:0.0:1767320083.911982:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320083.912010:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1010 00000100:00100000:2.0:1767320083.912020:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982980480:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320083.912050:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320083.912102:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982980480:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (587us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320083.912111:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1010 00000100:00100000:0.0:1767320083.912145:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982980480:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320083.915626:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982980352:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320083.915713:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320083.915954:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980352 00000100:00100000:2.0:1767320083.915963:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980352 00000100:00100000:2.0:1767320083.916043:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011 00000100:00100000:2.0:1767320083.916055:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982980352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320083.916085:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320083.916133:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982980352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (420us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320083.917715:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982980352:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320083.917940:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320083.918343:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011 00010000:00010000:1.0:1767320084.612503:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320085.636681:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320086.468372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320086.468382:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320086.468445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320086.468450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320086.468454:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320086.468462:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5329 00000100:00100000:2.0:1767320086.468463:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982980608:0@lo:400:kworker.0 00000100:00080000:0.0:1767320086.468466:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320086.468472:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320086.468491:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982980736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320086.468546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320086.468551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320086.468553:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320086.468556:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5329 00000100:00080000:0.0:1767320086.468559:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320086.468562:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320086.468574:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163982980992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320086.468583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320086.468586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320086.468588:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320086.468591:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5329 00000100:00080000:0.0:1767320086.468593:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320086.468596:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320086.468607:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320a80 x1853163982981120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320086.468614:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982980864:0@lo:400:kworker.0 00000100:00080000:0.0:1767320086.468632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320086.468635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320086.468637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320086.468640:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5329 00000100:00080000:0.0:1767320086.468643:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320086.468646:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320086.468647:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982981376:0@lo:400:kworker.0 00000100:00100000:0.0:1767320086.468655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320380 x1853163982981504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320086.468672:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320086.468693:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981376 00000100:00100000:0.0:1767320086.468700:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981376 00000100:00080000:1.0:1767320086.468732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320086.468747:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941 00000100:00100000:0.0:1767320086.468756:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982981376:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320086.468774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320086.468808:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982981248:0@lo:400:kworker.0 00000100:00100000:0.0:1767320086.468814:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982981376:12345-0@lo:400:kworker.0 Request processed in 60us (141us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320086.468831:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320086.468864:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980608 00000100:00100000:3.0:1767320086.468865:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982980736:0@lo:38:kworker.0 00000100:00100000:2.0:1767320086.468868:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980608 00000100:00100000:3.0:1767320086.468892:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320086.468915:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982980992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320086.468918:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981248 00000100:00100000:1.0:1767320086.468922:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981248 00000100:00100000:3.0:1767320086.468933:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320086.468946:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 942 00000100:00100000:2.0:1767320086.468958:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980736 00000100:00100000:2.0:1767320086.468961:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980736 00000100:00100000:2.0:1767320086.468981:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1502 00000100:00100000:2.0:1767320086.468988:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982980608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320086.469020:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982980608:12345-0@lo:400:kworker.0 Request processed in 33us (490us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320086.469026:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1502 00000100:00100000:2.0:1767320086.469041:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1984 00000100:00100000:2.0:1767320086.469045:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982980736:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320086.469061:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320086.469088:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982980736:12345-0@lo:38:kworker.0 Request processed in 42us (196us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320086.469097:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1984 00000100:00100000:1.0:1767320086.469139:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982981248:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320086.469178:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982981248:12345-0@lo:400:kworker.0 Request processed in 42us (346us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320086.469184:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 942 00000100:00100000:1.0:1767320086.469223:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982981248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320086.469249:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982981376:0@lo:400:kworker.0 00000100:00080000:1.0:1767320086.469260:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320086.469263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320086.469378:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323b80 x1853163982980608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320102 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320086.469388:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982980608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320086.470977:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941 00000100:00100000:3.0:1767320086.471560:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982981120:0@lo:38:kworker.0 00000100:00100000:3.0:1767320086.471662:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320086.471688:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982981504:0@lo:38:kworker.0 00000100:00100000:3.0:1767320086.471712:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320086.471755:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320086.471760:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320086.471775:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982980736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320086.471808:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981120 00000100:00100000:3.0:1767320086.471813:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981120 00000100:00100000:3.0:1767320086.471845:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 521 00000100:00100000:3.0:1767320086.471851:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982981120:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320086.471870:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320086.471911:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982981120:12345-0@lo:38:kworker.0 Request processed in 58us (248us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320086.471917:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 521 00000100:00100000:3.0:1767320086.471940:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980992 00000100:00100000:3.0:1767320086.471943:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980992 00000100:00100000:3.0:1767320086.471962:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981504 00000100:00100000:3.0:1767320086.471964:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981504 00000100:00100000:3.0:1767320086.471981:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1985 00000100:00100000:3.0:1767320086.471986:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982980992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320086.471995:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320086.472021:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982980992:12345-0@lo:38:kworker.0 Request processed in 35us (3087us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320086.472027:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1985 00000100:00100000:3.0:1767320086.472039:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1986 00000100:00100000:3.0:1767320086.472043:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982981504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320086.472053:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320086.472094:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982981504:12345-0@lo:38:kworker.0 Request processed in 49us (381us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320086.472100:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1986 00000100:00080000:3.0:1767320086.472132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320086.472136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320086.472140:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982980992:0@lo:38:kworker.0 00000100:00080000:3.0:1767320086.472148:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320086.472152:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320086.472154:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982981120:0@lo:38:kworker.0 00000100:00080000:3.0:1767320086.472161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320086.472164:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320086.472167:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982981504:0@lo:38:kworker.0 00000100:00100000:3.0:1767320086.472190:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320086.472231:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982980864 00000100:00100000:3.0:1767320086.472235:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982980864 00000100:00100000:3.0:1767320086.472313:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1503 00000100:00100000:3.0:1767320086.472321:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982980864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320086.472363:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982980864:12345-0@lo:400:kworker.0 Request processed in 44us (172us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320086.472372:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1503 00000100:00080000:3.0:1767320086.472391:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320086.472413:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322300 x1853163982980864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320102 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320086.472424:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982980864:0@lo:400:kworker.0 00010000:00010000:2.0:1767320086.660555:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320087.435571:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320087.435685:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661440 00000100:00100000:1.0:1767320087.435695:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661440 00000100:00100000:1.0:1767320087.435776:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1987 00000100:00100000:1.0:1767320087.435785:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915661440:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320087.435810:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320087.435888:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915661440:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (326us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320087.435895:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1987 00000100:00100000:1.0:1767320087.436762:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320087.436785:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915661696 offset 0 length 520: 4 00000100:00100000:3.0:1767320087.436883:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661568 00000100:00100000:3.0:1767320087.436891:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661568 00000100:00100000:3.0:1767320087.436953:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1504 00000100:00100000:3.0:1767320087.436963:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915661568:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320087.437970:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915661568:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1003us (1217us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320087.437981:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1504 00000100:00080000:1.0:1767320087.438069:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320087.685609:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320088.709739:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320089.028643:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982981632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320089.028713:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320089.028860:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981632 00000100:00100000:3.0:1767320089.028870:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981632 00000100:00100000:3.0:1767320089.028939:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1012 00000100:00100000:3.0:1767320089.028950:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982981632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320089.028980:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320089.029026:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982981632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320089.029033:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1012 00000100:00100000:2.0:1767320089.029199:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982981632:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320089.029434:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320089.029665:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320089.029677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982981760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320089.029715:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320089.029808:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982981760 00000100:00100000:2.0:1767320089.029813:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982981760 00000100:00100000:2.0:1767320089.029863:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013 00000100:00100000:2.0:1767320089.029870:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982981760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320089.029891:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320089.029928:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982981760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (213us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320089.029934:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013 00000100:00080000:0.0:1767320089.029991:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320089.030054:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982981760:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320089.734078:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320090.564408:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320065/real 1767320065] req@ffff93463c625180 x1853163982975360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320090 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320090.564445:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c625180 x1853163982975360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320090 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320090.564471:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320090.564497:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320090.564502:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982975360:0@lo:250:kworker.0 00000100:00080000:0.0:1767320090.565197:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.565203:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320090.565206:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.565210:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5314 00000100:00080000:0.0:1767320090.565214:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.565221:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320090.565253:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982981888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.565345:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.565349:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.565352:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.565356:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5335 00000100:00080000:0.0:1767320090.565359:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.565364:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320090.565371:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982982016:0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.565380:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321500 x1853163982982144/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.565411:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.565415:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.565417:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.565422:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5335 00000100:00080000:0.0:1767320090.565425:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320090.565429:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320090.565432:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320090.565448:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322300 x1853163982982400/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.565458:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.565462:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.565464:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.565468:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5335 00000100:00080000:0.0:1767320090.565470:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.565474:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320090.565487:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320380 x1853163982982528/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320090.565514:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320090.565518:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320090.565520:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320090.565523:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5335 00000100:00080000:0.0:1767320090.565525:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320090.565529:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320090.565538:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982982912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320090.565554:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982982272:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.565573:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320090.565584:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320090.565596:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982982784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.565597:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320090.565605:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982982656:0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.565630:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320090.565767:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982016 00000100:00100000:2.0:1767320090.565777:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982016 00000100:00100000:2.0:1767320090.565940:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982272 00000100:00100000:2.0:1767320090.565944:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982272 00000100:00100000:2.0:1767320090.565970:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1505 00000100:00100000:2.0:1767320090.565978:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982982016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320090.566057:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1506 00000100:00100000:0.0:1767320090.566146:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982656 00000100:00100000:0.0:1767320090.566151:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982656 00000100:00100000:0.0:1767320090.566182:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 943 00000100:00100000:0.0:1767320090.566188:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982982656:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320090.566492:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320090.566497:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982982016:12345-0@lo:400:kworker.0 Request processed in 517us (1066us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320090.566505:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1505 00000100:00080000:2.0:1767320090.566527:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320090.566554:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982981888:0@lo:250:kworker.0 00000400:00000100:2.0:1767320090.566578:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982981888 offset 0 length 520: 4 00000100:00100000:2.0:1767320090.566599:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982982144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320090.566622:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320090.566648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982982400:0@lo:38:kworker.0 00010000:00080000:1.0:1767320090.566653:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321c00 x1853163982982016/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320106 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320090.566672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320090.566674:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982982016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320090.566705:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982982528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320090.566731:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320090.566750:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982982912:0@lo:38:kworker.0 00000100:00100000:1.0:1767320090.566760:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982528 00000100:00100000:1.0:1767320090.566766:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982528 00000100:00100000:2.0:1767320090.566772:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320090.566793:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 522 00000100:00100000:1.0:1767320090.566799:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982982528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.566815:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320090.566990:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982982272:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320090.567009:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982784 00000100:00100000:2.0:1767320090.567012:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982784 00000100:00100000:2.0:1767320090.567037:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1767320090.567043:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982982784:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.568608:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982982656:12345-0@lo:400:kworker.0 Request processed in 2412us (2968us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320090.568621:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 943 00000100:00100000:1.0:1767320090.568621:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982982528:12345-0@lo:38:kworker.0 Request processed in 1818us (1886us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.568634:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 522 00000100:00100000:0.0:1767320090.568680:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982982656:0@lo:400:kworker.0 00000100:00100000:0.0:1767320090.568741:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982144 00000100:00100000:0.0:1767320090.568746:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982144 00000100:00100000:0.0:1767320090.568762:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982400 00000100:00100000:0.0:1767320090.568764:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982400 00000100:00100000:0.0:1767320090.568771:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982982912 00000100:00100000:0.0:1767320090.568773:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982982912 00000100:00080000:1.0:1767320090.568883:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320090.568889:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320090.568892:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982982528:0@lo:38:kworker.0 00000100:00100000:1.0:1767320090.568921:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1988 00000100:00100000:1.0:1767320090.568927:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982144:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.568945:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.568985:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982144:12345-0@lo:38:kworker.0 Request processed in 56us (2362us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.568992:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1988 00000100:00100000:1.0:1767320090.569006:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989 00000100:00100000:1.0:1767320090.569010:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.569019:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.569044:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982400:12345-0@lo:38:kworker.0 Request processed in 32us (2372us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.569050:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989 00000100:00100000:1.0:1767320090.569061:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1990 00000100:00100000:1.0:1767320090.569065:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320090.569074:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320090.569096:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982982912:12345-0@lo:38:kworker.0 Request processed in 30us (2325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320090.569102:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1990 00000100:00080000:1.0:1767320090.569124:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320090.569129:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320090.569132:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982982144:0@lo:38:kworker.0 00000100:00080000:1.0:1767320090.569142:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320090.569145:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320090.569148:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982982400:0@lo:38:kworker.0 00000100:00080000:1.0:1767320090.569155:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320090.569158:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320090.569161:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982982912:0@lo:38:kworker.0 00000100:00100000:2.0:1767320090.569443:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982982784:12345-0@lo:400:kworker.0 Request processed in 2399us (2949us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320090.569453:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944 00000100:00100000:1.0:1767320090.569482:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982982784:0@lo:400:kworker.0 00000100:00100000:3.0:1767320090.569639:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982982272:12345-0@lo:400:kworker.0 Request processed in 3573us (4062us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320090.569648:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1506 00010000:00080000:0.0:1767320090.569713:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322d80 x1853163982982272/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320106 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320090.569726:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982982272:0@lo:400:kworker.0 00010000:00010000:1.0:1767320090.759293:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320091.780774:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320092.558149:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320092.558249:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661952 00000100:00100000:3.0:1767320092.558259:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661952 00000100:00100000:3.0:1767320092.558347:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1507 00000100:00100000:3.0:1767320092.558357:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915661952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320092.558443:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915661952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (303us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320092.558483:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320092.558514:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1507 00000100:00100000:0.0:1767320092.560999:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320092.561107:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915661824 00000100:00100000:3.0:1767320092.561118:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915661824 00000100:00100000:3.0:1767320092.561163:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1991 00000100:00100000:3.0:1767320092.561171:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915661824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320092.561193:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320092.561263:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915661824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 91us (270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320092.561298:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1991 00010000:00010000:2.0:1767320092.804502:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320093.830105:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320094.152259:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982983040:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320094.152363:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320094.152572:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983040 00000100:00100000:2.0:1767320094.152583:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983040 00000100:00100000:2.0:1767320094.152661:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1014 00000100:00100000:2.0:1767320094.152670:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982983040:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320094.152703:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320094.152749:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982983040:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (386us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320094.152756:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1014 00000100:00100000:2.0:1767320094.152797:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982983040:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320094.152983:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320094.156508:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982983168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320094.156584:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320094.156697:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983168 00000100:00100000:1.0:1767320094.156707:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983168 00000100:00080000:0.0:1767320094.156769:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320094.156788:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1015 00000100:00100000:1.0:1767320094.156799:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982983168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320094.156831:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320094.156892:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982983168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (307us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320094.156901:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1015 00000100:00100000:0.0:1767320094.156979:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982983168:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320094.852601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320095.684451:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320095.684462:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320095.684561:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982983296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.684640:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320095.684734:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983296 00000100:00100000:3.0:1767320095.684744:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983296 00000100:00100000:3.0:1767320095.684856:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1508 00000100:00100000:3.0:1767320095.684866:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982983296:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320095.687686:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320095.687693:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320095.687697:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320095.687702:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5339 00000100:00080000:0.0:1767320095.687706:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320095.687711:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320095.687744:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163982983424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320095.687780:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320095.687782:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320095.687783:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320095.687785:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5339 00000100:00080000:0.0:1767320095.687787:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320095.687789:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320095.687795:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163982983680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320095.687800:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320095.687803:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320095.687804:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320095.687806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5339 00000100:00080000:0.0:1767320095.687807:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320095.687809:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320095.687814:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322300 x1853163982983808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320095.687910:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982983936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.687962:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320095.687986:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320095.687990:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982983552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.688003:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320095.688031:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983936 00000100:00100000:2.0:1767320095.688036:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983936 00000100:00100000:2.0:1767320095.688068:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945 00000100:00100000:2.0:1767320095.688073:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982983936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320095.688106:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982983936:12345-0@lo:400:kworker.0 Request processed in 33us (144us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320095.688110:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945 00000100:00100000:2.0:1767320095.688136:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982983936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320095.688551:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982983296:12345-0@lo:400:kworker.0 Request processed in 3684us (3907us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320095.688562:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1508 00000100:00100000:3.0:1767320095.688580:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983552 00000100:00100000:3.0:1767320095.688584:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983552 00000100:00080000:1.0:1767320095.688643:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320095.688665:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982983424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320095.688700:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320095.688721:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982983680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320095.688740:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320095.688756:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982983808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320095.688778:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320095.688803:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983424 00000100:00100000:1.0:1767320095.688807:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983424 00000100:00100000:1.0:1767320095.688824:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983680 00000100:00100000:1.0:1767320095.688827:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983680 00000100:00100000:1.0:1767320095.688838:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1992 00000100:00100000:1.0:1767320095.688843:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982983424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320095.688862:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320095.688895:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982983424:12345-0@lo:38:kworker.0 Request processed in 51us (194us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320095.688902:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1992 00000100:00100000:1.0:1767320095.688916:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993 00000100:00100000:1.0:1767320095.688920:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982983680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320095.688929:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320095.688953:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982983680:12345-0@lo:38:kworker.0 Request processed in 31us (212us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320095.688959:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993 00000100:00100000:1.0:1767320095.688979:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982983808 00000100:00100000:1.0:1767320095.688983:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982983808 00000100:00100000:1.0:1767320095.689001:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 523 00000100:00100000:1.0:1767320095.689006:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982983808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320095.689016:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320095.689041:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982983808:12345-0@lo:38:kworker.0 Request processed in 35us (263us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320095.689047:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 523 00000100:00080000:1.0:1767320095.689078:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320095.689083:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320095.689086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982983424:0@lo:38:kworker.0 00000100:00080000:1.0:1767320095.689095:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320095.689098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320095.689100:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982983680:0@lo:38:kworker.0 00000100:00080000:1.0:1767320095.689108:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320095.689111:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320095.689114:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982983808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320095.689162:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982984064:0@lo:400:kworker.0 00000100:00100000:1.0:1767320095.689190:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320095.689231:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984064 00000100:00100000:1.0:1767320095.689235:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984064 00000100:00100000:1.0:1767320095.689259:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 946 00000100:00100000:1.0:1767320095.689265:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982984064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320095.689319:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982984064:12345-0@lo:400:kworker.0 Request processed in 54us (129us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320095.689329:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 946 00000100:00100000:1.0:1767320095.689353:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982984064:0@lo:400:kworker.0 00010000:00080000:2.0:1767320095.691894:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322a00 x1853163982983296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320111 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320095.691911:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982983296:0@lo:400:kworker.0 00000100:00080000:0.0:1767320095.692113:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320095.692117:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320095.692120:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320095.692124:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5339 00000100:00080000:0.0:1767320095.692126:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320095.692130:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320095.692158:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321c00 x1853163982984192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320095.692323:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982984192:0@lo:38:kworker.0 00000100:00100000:2.0:1767320095.692366:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320095.692425:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984192 00000100:00100000:0.0:1767320095.692430:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984192 00000100:00100000:0.0:1767320095.692460:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1994 00000100:00100000:0.0:1767320095.692464:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982984192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320095.692478:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320095.692632:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982984192:12345-0@lo:38:kworker.0 Request processed in 166us (264us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320095.692638:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1994 00000100:00100000:3.0:1767320095.693214:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1509 00000100:00100000:3.0:1767320095.693225:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982983552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320095.693297:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982983552:12345-0@lo:400:kworker.0 Request processed in 74us (5290us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320095.693303:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1509 00010000:00080000:3.0:1767320095.693348:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320700 x1853163982983552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320111 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320095.693360:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982983552:0@lo:400:kworker.0 00000100:00080000:2.0:1767320095.694648:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320095.694656:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320095.694659:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982984192:0@lo:38:kworker.0 00010000:00010000:2.0:1767320095.882673:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320096.900688:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320097.676329:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320097.676803:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662208 00000100:00100000:0.0:1767320097.676815:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662208 00000100:00100000:0.0:1767320097.676926:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1510 00000100:00100000:0.0:1767320097.676937:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915662208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320097.677023:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915662208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (703us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320097.677031:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1510 00000100:00080000:1.0:1767320097.677671:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320097.679629:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320097.679823:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662080 00000100:00100000:0.0:1767320097.679835:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662080 00000100:00100000:0.0:1767320097.679886:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1995 00000100:00100000:0.0:1767320097.679895:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915662080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320097.679921:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320097.679984:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915662080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 89us (365us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320097.679992:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1995 00010000:00010000:2.0:1767320097.924738:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320098.949576:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320099.268813:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320099.268841:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982984320:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320099.268913:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320099.269016:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984320 00000100:00100000:2.0:1767320099.269025:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984320 00000100:00100000:2.0:1767320099.269095:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016 00000100:00100000:2.0:1767320099.269104:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982984320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320099.269135:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320099.269187:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982984320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (273us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320099.269194:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016 00000100:00100000:2.0:1767320099.269240:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982984320:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320099.269304:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320099.269408:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982984448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320099.269437:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320099.269646:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984448 00000100:00100000:3.0:1767320099.269651:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984448 00000100:00100000:3.0:1767320099.269693:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017 00000100:00100000:3.0:1767320099.269700:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982984448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320099.269717:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320099.269749:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982984448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320099.269756:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017 00000100:00080000:1.0:1767320099.269813:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320099.269850:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982984448:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320099.972504:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320100.805414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320100.805425:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320100.805489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320100.805494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320100.805507:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320100.805516:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5344 00000100:00080000:2.0:1767320100.805520:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320100.805529:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320100.805548:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320100.805565:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163982984704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320100.805570:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982984576:0@lo:400:kworker.0 00000100:00080000:2.0:1767320100.805631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320100.805635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320100.805637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320100.805641:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5344 00000100:00080000:2.0:1767320100.805645:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320100.805649:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320100.805657:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320100.805666:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56680 x1853163982984960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320100.805676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320100.805679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320100.805682:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320100.805686:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5344 00000100:00080000:2.0:1767320100.805688:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320100.805692:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320100.805705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56d80 x1853163982985088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320100.805727:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982984832:0@lo:400:kworker.0 00000100:00080000:2.0:1767320100.805738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320100.805743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320100.805745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320100.805750:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5344 00000100:00080000:2.0:1767320100.805755:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320100.805756:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320100.805763:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320100.805777:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320100.805780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54380 x1853163982985472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320100.805785:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982985216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320100.805816:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320100.805819:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982985344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320100.805852:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985216 00000100:00100000:2.0:1767320100.805857:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320100.805861:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985216 00000100:00100000:2.0:1767320100.805907:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985344 00000100:00100000:2.0:1767320100.805912:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985344 00000100:00100000:1.0:1767320100.805938:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947 00000100:00100000:2.0:1767320100.805946:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948 00000100:00100000:1.0:1767320100.805952:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982985216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320100.805959:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982985344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320100.806009:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982985344:12345-0@lo:400:kworker.0 Request processed in 53us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320100.806015:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982985216:12345-0@lo:400:kworker.0 Request processed in 66us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320100.806020:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948 00000100:00100000:1.0:1767320100.806028:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947 00000100:00100000:1.0:1767320100.806075:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982985216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320100.806322:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982985344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320100.806445:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982984704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320100.806480:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320100.806522:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982984960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320100.806554:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320100.806579:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982985088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320100.806611:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320100.806652:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982985472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320100.806672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320100.806702:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985088 00000100:00100000:0.0:1767320100.806707:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985088 00000100:00100000:0.0:1767320100.806729:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 524 00000100:00100000:0.0:1767320100.806734:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982985088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320100.806755:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320100.806797:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982985088:12345-0@lo:38:kworker.0 Request processed in 62us (184us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320100.806804:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 524 00000100:00080000:0.0:1767320100.806833:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320100.806833:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984576 00000100:00100000:3.0:1767320100.806839:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984576 00000100:00080000:0.0:1767320100.806841:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320100.806848:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982985088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320100.806873:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984832 00000100:00100000:0.0:1767320100.806878:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984832 00000100:00100000:3.0:1767320100.806898:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984704 00000100:00100000:3.0:1767320100.806903:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984704 00000100:00100000:0.0:1767320100.806904:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982984960 00000100:00100000:0.0:1767320100.806909:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982984960 00000100:00100000:3.0:1767320100.806919:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985472 00000100:00100000:3.0:1767320100.806924:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985472 00000100:00100000:0.0:1767320100.806928:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1511 00000100:00100000:0.0:1767320100.806937:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982984576:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320100.806944:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1512 00000100:00100000:3.0:1767320100.806954:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982984832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320100.806978:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982984576:12345-0@lo:400:kworker.0 Request processed in 41us (1321us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320100.806986:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1511 00000100:00100000:0.0:1767320100.807005:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1996 00000100:00100000:3.0:1767320100.807008:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982984832:12345-0@lo:400:kworker.0 Request processed in 55us (1252us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320100.807014:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982984704:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320100.807021:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1512 00010000:00020000:0.0:1767320100.807030:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320100.807040:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1997 00000100:00100000:3.0:1767320100.807045:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982984960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320100.807056:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320100.807064:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982984704:12345-0@lo:38:kworker.0 Request processed in 51us (583us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320100.807086:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982984960:12345-0@lo:38:kworker.0 Request processed in 39us (530us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320100.807092:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1997 00000100:00100000:3.0:1767320100.807103:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1998 00000100:00100000:3.0:1767320100.807107:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982985472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320100.807115:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320100.807137:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982985472:12345-0@lo:38:kworker.0 Request processed in 29us (465us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320100.807143:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1998 00000100:00080000:3.0:1767320100.807158:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320100.807189:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57480 x1853163982984832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320116 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320100.807199:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982984832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320100.807494:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1996 00000100:00080000:0.0:1767320100.807668:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320100.807674:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320100.807677:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982984704:0@lo:38:kworker.0 00000100:00080000:0.0:1767320100.807690:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320100.807693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320100.807696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982984960:0@lo:38:kworker.0 00000100:00080000:0.0:1767320100.807703:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320100.807706:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320100.807709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982985472:0@lo:38:kworker.0 00010000:00080000:2.0:1767320100.807807:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57100 x1853163982984576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320116 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320100.807817:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982984576:0@lo:400:kworker.0 00010000:00010000:2.0:1767320100.996565:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320102.021039:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320102.800459:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320102.800726:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320102.800819:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662464 00000100:00100000:3.0:1767320102.800833:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662464 00000100:00100000:3.0:1767320102.800916:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662336 00000100:00100000:3.0:1767320102.800919:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662336 00000100:00100000:3.0:1767320102.800954:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1513 00000100:00100000:3.0:1767320102.800968:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915662464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320102.801075:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915662464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 111us (644us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320102.801083:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1513 00000100:00100000:3.0:1767320102.801107:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1999 00000100:00100000:3.0:1767320102.801111:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915662336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320102.801139:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320102.801169:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915662336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (442us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320102.801175:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1999 00000100:00080000:2.0:1767320102.801333:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320103.044776:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320104.068775:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320104.388675:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982985600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320104.388778:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320104.388930:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985600 00000100:00100000:0.0:1767320104.388949:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985600 00000100:00100000:0.0:1767320104.389051:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1018 00000100:00100000:0.0:1767320104.389063:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982985600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320104.389105:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767320104.389127:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320104.389224:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982985728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320104.389262:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320104.389356:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985728 00000100:00100000:2.0:1767320104.389360:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985728 00000100:00100000:2.0:1767320104.389392:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019 00000100:00100000:2.0:1767320104.389399:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982985728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320104.389421:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320104.389471:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982985728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (207us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320104.389480:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019 00000100:00080000:2.0:1767320104.389504:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320104.389545:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982985728:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320104.393101:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982985600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4026us (4308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320104.393118:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1018 00000100:00100000:3.0:1767320104.393609:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982985600:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320105.092990:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320105.924422:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320105.924438:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320105.924532:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320105.924537:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320105.924542:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320105.924553:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5349 00000100:00080000:2.0:1767320105.924560:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320105.924569:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320105.924595:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54700 x1853163982985984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320105.924646:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320105.924650:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320105.924652:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320105.924656:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5349 00000100:00080000:2.0:1767320105.924659:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320105.924663:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320105.924672:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55880 x1853163982986240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320105.924682:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320105.924685:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320105.924687:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320105.924690:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5349 00000100:00080000:2.0:1767320105.924693:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320105.924696:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320105.924714:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163982986368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320105.924736:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320105.924739:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320105.924741:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320105.924745:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5349 00000100:00080000:2.0:1767320105.924747:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320105.924750:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320105.924758:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54380 x1853163982986752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320105.924786:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982986112:0@lo:400:kworker.0 00000100:00100000:2.0:1767320105.924875:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320105.924965:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320105.924972:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982986496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320105.924981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982985856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320105.925002:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320105.925026:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320105.925047:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986112 00000100:00100000:0.0:1767320105.925053:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320105.925056:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986112 00000100:00100000:0.0:1767320105.925057:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982986624:0@lo:400:kworker.0 00000100:00100000:0.0:1767320105.925078:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320105.925103:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985856 00000100:00100000:0.0:1767320105.925106:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985856 00000100:00100000:2.0:1767320105.925120:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1514 00000100:00100000:2.0:1767320105.925129:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982986112:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320105.925137:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1515 00000100:00100000:0.0:1767320105.925142:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982985856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320105.925186:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982986112:12345-0@lo:400:kworker.0 Request processed in 57us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320105.925187:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982985856:12345-0@lo:400:kworker.0 Request processed in 45us (160us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320105.925194:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1514 00000100:00080000:2.0:1767320105.925224:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320105.925257:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55c00 x1853163982986112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320121 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320105.925299:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982986112:0@lo:400:kworker.0 00000100:00100000:3.0:1767320105.925337:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986496 00000100:00100000:3.0:1767320105.925342:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986496 00000100:00100000:3.0:1767320105.925372:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986624 00000100:00100000:3.0:1767320105.925375:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986624 00000100:00100000:3.0:1767320105.925396:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 949 00000100:00100000:3.0:1767320105.925413:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982986496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320105.925457:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982986496:12345-0@lo:400:kworker.0 Request processed in 54us (453us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320105.925463:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 949 00000100:00100000:3.0:1767320105.925478:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 950 00000100:00100000:3.0:1767320105.925484:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982986624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320105.925503:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982985984:0@lo:38:kworker.0 00000100:00100000:3.0:1767320105.925508:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982986624:12345-0@lo:400:kworker.0 Request processed in 25us (430us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320105.925514:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 950 00000100:00100000:1.0:1767320105.925535:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320105.925646:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982985984 00000100:00100000:2.0:1767320105.925650:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982985984 00000100:00100000:1.0:1767320105.925682:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982986240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320105.925812:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320105.925829:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982986368:0@lo:38:kworker.0 00000100:00100000:1.0:1767320105.925851:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320105.925870:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982986752:0@lo:38:kworker.0 00000100:00100000:1.0:1767320105.925896:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320105.925933:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986368 00000100:00100000:1.0:1767320105.925937:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986368 00000100:00100000:1.0:1767320105.925961:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 525 00000100:00100000:1.0:1767320105.925967:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982986368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320105.925990:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320105.926023:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982986368:12345-0@lo:38:kworker.0 Request processed in 56us (171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320105.926030:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 525 00000100:00100000:1.0:1767320105.926048:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986240 00000100:00100000:1.0:1767320105.926052:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986240 00000100:00100000:0.0:1767320105.927649:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1515 00000100:00100000:0.0:1767320105.927700:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986752 00000100:00100000:0.0:1767320105.927709:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986752 00000100:00100000:0.0:1767320105.927738:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2000 00000100:00100000:0.0:1767320105.927745:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982986752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320105.927769:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320105.927858:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001 00000100:00100000:2.0:1767320105.927865:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982985984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320105.927876:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320105.928055:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982985984:12345-0@lo:38:kworker.0 Request processed in 188us (2517us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320105.928062:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001 00000100:00100000:2.0:1767320105.928121:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982986624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320105.928142:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982986496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320105.928190:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2002 00000100:00100000:1.0:1767320105.928196:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982986240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320105.928207:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:2.0:1767320105.928249:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57b80 x1853163982985856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320121 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320105.928260:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982985856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320105.928398:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982986240:12345-0@lo:38:kworker.0 Request processed in 200us (2585us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320105.928415:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2002 00000100:00080000:1.0:1767320105.928466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320105.928472:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320105.928594:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982985984:0@lo:38:kworker.0 00000100:00080000:1.0:1767320105.928610:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320105.928614:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320105.928617:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982986240:0@lo:38:kworker.0 00000100:00080000:1.0:1767320105.928627:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320105.928631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320105.928634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982986368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320105.929662:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982986752:12345-0@lo:38:kworker.0 Request processed in 1910us (3757us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320105.929675:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2000 00000100:00080000:1.0:1767320105.929741:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320105.929745:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320105.929749:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982986752:0@lo:38:kworker.0 00010000:00010000:1.0:1767320106.117968:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320107.140517:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320107.917423:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320107.917613:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320107.918565:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662720 00000100:00100000:0.0:1767320107.918582:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662720 00000100:00100000:0.0:1767320107.918684:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662592 00000100:00100000:0.0:1767320107.918688:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662592 00000100:00100000:0.0:1767320107.918718:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1516 00000100:00100000:0.0:1767320107.918729:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915662720:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320107.918828:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915662720:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (1422us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320107.918836:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1516 00000100:00100000:0.0:1767320107.918856:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2003 00000100:00100000:0.0:1767320107.918860:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915662592:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320107.918886:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320107.918912:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915662592:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (1300us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320107.918918:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2003 00000100:00080000:2.0:1767320107.919050:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320108.165536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320109.189300:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320109.508777:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982986880:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320109.508883:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320109.508897:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982987008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320109.508997:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320109.509049:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982986880 00000100:00100000:1.0:1767320109.509061:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982986880 00000100:00100000:1.0:1767320109.509149:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987008 00000100:00100000:1.0:1767320109.509153:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987008 00000100:00100000:1.0:1767320109.509175:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020 00000100:00100000:1.0:1767320109.509185:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982986880:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320109.509221:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320109.509227:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767320109.509233:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320109.509345:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1021 00000100:00100000:3.0:1767320109.509371:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982987008:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320109.509379:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982986880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 182us (483us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320109.509390:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020 00002000:00000024:3.0:1767320109.509411:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320109.509414:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982986880:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320109.509490:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982987008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 133us (492us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320109.509501:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1021 00000100:00100000:2.0:1767320109.509538:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982987008:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320110.214129:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320111.044751:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320111.044761:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320111.044827:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320111.044831:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320111.044834:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320111.044843:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5354 00000100:00080000:0.0:1767320111.044848:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320111.044855:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320111.044878:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b800 x1853163982987264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320111.045179:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320111.045194:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982987136:0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.045258:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320111.045377:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987136 00000100:00100000:3.0:1767320111.045385:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987136 00000100:00100000:3.0:1767320111.045436:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1517 00000100:00100000:3.0:1767320111.045445:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982987136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.045511:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163982987136:12345-0@lo:400:kworker.0 Request processed in 67us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320111.045518:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1517 00010000:00080000:3.0:1767320111.045557:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321500 x1853163982987136/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320127 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320111.045571:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163982987136:0@lo:400:kworker.0 00000100:00080000:2.0:1767320111.045872:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320111.046197:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320111.046202:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320111.046204:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320111.046209:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5354 00000100:00080000:0.0:1767320111.046212:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320111.046216:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320111.046239:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190a300 x1853163982987520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320111.046250:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320111.046253:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320111.046254:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320111.046257:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5354 00000100:00080000:0.0:1767320111.046259:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320111.046263:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320111.046302:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b480 x1853163982987648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320111.046338:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320111.046342:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320111.046344:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320111.046347:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5354 00000100:00080000:0.0:1767320111.046350:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320111.046353:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320111.046364:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631908000 x1853163982988032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320111.046492:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982987264:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.046534:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320111.046578:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982987520:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.046599:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320111.046769:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982987648:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.046796:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320111.046816:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982988032:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.046834:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320111.046918:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982987776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.046959:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320111.046995:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320111.047002:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982987392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.047188:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320111.047213:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320111.047221:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982987904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.047246:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320111.047565:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987776 00000100:00100000:1.0:1767320111.047572:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987776 00000100:00100000:0.0:1767320111.047732:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987264 00000100:00100000:0.0:1767320111.047736:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987264 00000100:00100000:0.0:1767320111.047756:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987520 00000100:00100000:0.0:1767320111.047759:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987520 00000100:00100000:0.0:1767320111.047767:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988032 00000100:00100000:0.0:1767320111.047770:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988032 00000100:00100000:0.0:1767320111.047776:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987392 00000100:00100000:0.0:1767320111.047779:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987392 00000100:00100000:0.0:1767320111.047799:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1518 00000100:00100000:0.0:1767320111.047808:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982987392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320111.047865:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982987392:12345-0@lo:400:kworker.0 Request processed in 57us (676us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320111.047872:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1518 00000100:00100000:0.0:1767320111.047889:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1767320111.047893:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982987264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320111.047924:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320111.048184:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982987264:12345-0@lo:38:kworker.0 Request processed in 288us (1649us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.048191:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1767320111.048205:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2005 00000100:00100000:0.0:1767320111.048208:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982987520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320111.048219:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320111.048242:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982987520:12345-0@lo:38:kworker.0 Request processed in 32us (1641us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.048247:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2005 00000100:00100000:0.0:1767320111.048257:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2006 00000100:00100000:0.0:1767320111.048260:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982988032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320111.048268:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320111.048392:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987904 00000100:00100000:1.0:1767320111.048397:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987904 00000100:00100000:1.0:1767320111.048422:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951 00000100:00100000:1.0:1767320111.048428:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982987776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.048592:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982987648 00000100:00100000:3.0:1767320111.048597:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982987648 00000100:00100000:3.0:1767320111.048714:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 526 00000100:00100000:3.0:1767320111.048719:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982987648:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320111.048730:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320111.048762:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982987648:12345-0@lo:38:kworker.0 Request processed in 42us (1966us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320111.048768:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 526 00000100:00100000:3.0:1767320111.048791:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952 00000100:00100000:3.0:1767320111.048798:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982987904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320111.048831:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982987904:12345-0@lo:400:kworker.0 Request processed in 34us (1584us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320111.048837:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952 00000100:00080000:3.0:1767320111.048927:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320111.048932:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00080000:2.0:1767320111.049159:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320111.049164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320111.049167:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320111.049194:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982987904:0@lo:400:kworker.0 00010000:00080000:2.0:1767320111.049229:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190ad80 x1853163982987392/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320127 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320111.049241:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982987392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320111.049777:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982987776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320111.054268:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982988032:12345-0@lo:38:kworker.0 Request processed in 5996us (7423us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320111.054306:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2006 00000100:00100000:3.0:1767320111.054420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982987264:0@lo:38:kworker.0 00000100:00080000:3.0:1767320111.054466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320111.054471:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320111.054474:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982987520:0@lo:38:kworker.0 00000100:00080000:3.0:1767320111.054488:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320111.054491:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320111.054494:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982987648:0@lo:38:kworker.0 00000100:00080000:3.0:1767320111.054504:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320111.054507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320111.054510:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982988032:0@lo:38:kworker.0 00000100:00100000:1.0:1767320111.055407:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982987776:12345-0@lo:400:kworker.0 Request processed in 6973us (8442us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320111.055420:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951 00010000:00010000:2.0:1767320111.236609:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767320112.015713:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915662848 offset 0 length 520: 4 00000100:00100000:1.0:1767320112.015762:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320112.018331:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663104 00000100:00100000:0.0:1767320112.018349:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663104 00000100:00100000:0.0:1767320112.018544:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1519 00000100:00100000:0.0:1767320112.018555:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915663104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320112.018653:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915663104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (2890us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320112.018661:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1519 00000100:00100000:0.0:1767320112.020067:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:2.0:1767320112.022827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320112.023514:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915662976 00000100:00100000:3.0:1767320112.023536:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915662976 00000100:00100000:3.0:1767320112.023711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2007 00000100:00100000:3.0:1767320112.023722:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915662976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320112.023768:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320112.023885:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915662976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 163us (3830us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320112.023894:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2007 00010000:00010000:1.0:1767320112.267645:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320113.284516:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320114.308623:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320114.628868:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320114.628903:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982988160:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320114.628998:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320114.629166:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988160 00000100:00100000:2.0:1767320114.629180:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988160 00000100:00080000:0.0:1767320114.629361:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320114.629470:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1022 00000100:00100000:2.0:1767320114.629483:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982988160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320114.629537:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320114.629931:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982988160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 451us (931us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320114.629941:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1022 00000100:00100000:2.0:1767320114.630124:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982988288:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320114.630164:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320114.630458:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988288 00000100:00100000:1.0:1767320114.630467:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988288 00000100:00100000:1.0:1767320114.630523:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1767320114.630530:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982988288:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320114.630854:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320114.630881:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320114.630952:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982988288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 422us (786us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320114.630960:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023 00000100:00100000:2.0:1767320114.631055:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982988288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320114.631388:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982988160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320115.140564:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320090/real 1767320090] req@ffff934604320000 x1853163982981888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320115 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320115.140613:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604320000 x1853163982981888/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320115 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320115.140639:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320115.140677:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320115.140681:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982981888:0@lo:250:kworker.0 00000100:00080000:0.0:1767320115.140847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320115.140853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320115.140857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320115.140863:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5339 00000100:00080000:0.0:1767320115.140867:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320115.140875:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320115.140923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909180 x1853163982988416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320115.141009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320115.141013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320115.141016:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320115.141021:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5359 00000100:00080000:0.0:1767320115.141023:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320115.141027:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320115.141044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190a680 x1853163982988672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320115.141076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320115.141080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320115.141082:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320115.141086:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5359 00000100:00080000:0.0:1767320115.141089:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320115.141093:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320115.141104:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909500 x1853163982988928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320115.141113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320115.141117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320115.141118:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320115.141122:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5359 00000100:00080000:0.0:1767320115.141124:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320115.141128:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320115.141145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909880 x1853163982989056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320115.141155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320115.141159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320115.141161:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320115.141164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5359 00000100:00080000:0.0:1767320115.141166:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320115.141170:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320115.141180:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190aa00 x1853163982989184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320115.141212:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320115.141227:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982988800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.141372:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320115.141634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982988544:0@lo:400:kworker.0 00000100:00100000:2.0:1767320115.141681:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320115.141828:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988800 00000100:00100000:0.0:1767320115.141839:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988800 00000100:00100000:0.0:1767320115.141911:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988544 00000100:00100000:0.0:1767320115.141915:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988544 00000100:00100000:0.0:1767320115.141941:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1520 00000100:00100000:0.0:1767320115.141950:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982988800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.142014:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982988800:12345-0@lo:400:kworker.0 Request processed in 65us (642us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320115.142022:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1520 00000100:00100000:0.0:1767320115.142040:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1521 00000100:00100000:0.0:1767320115.142046:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982988544:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320115.142074:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982988544:12345-0@lo:400:kworker.0 Request processed in 29us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320115.142080:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1521 00010000:00080000:0.0:1767320115.142118:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190b100 x1853163982988800/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320131 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320115.142134:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982988800:0@lo:400:kworker.0 00000100:00100000:3.0:1767320115.142454:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982988416:0@lo:250:kworker.0 00000400:00000100:3.0:1767320115.142487:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982988416 offset 0 length 520: 4 00000100:00100000:3.0:1767320115.142511:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982988672:0@lo:38:kworker.0 00000100:00100000:3.0:1767320115.142536:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320115.142566:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982988928:0@lo:38:kworker.0 00000100:00100000:3.0:1767320115.142588:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320115.142605:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982989056:0@lo:38:kworker.0 00000100:00100000:3.0:1767320115.142630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320115.142648:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982989184:0@lo:38:kworker.0 00000100:00100000:0.0:1767320115.142761:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988672 00000100:00100000:0.0:1767320115.142766:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988672 00000100:00100000:0.0:1767320115.142788:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982988928 00000100:00100000:0.0:1767320115.142790:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982988928 00000100:00100000:0.0:1767320115.142802:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2008 00000100:00100000:0.0:1767320115.142807:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982988672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320115.142830:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320115.142865:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982988672:12345-0@lo:38:kworker.0 Request processed in 56us (327us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.142872:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2008 00000100:00100000:0.0:1767320115.142883:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2009 00000100:00100000:0.0:1767320115.142887:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982988928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320115.142895:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320115.142916:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982988928:12345-0@lo:38:kworker.0 Request processed in 27us (327us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.142921:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2009 00000100:00100000:3.0:1767320115.143070:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320115.143108:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320115.143114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320115.143117:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982988672:0@lo:38:kworker.0 00000100:00080000:3.0:1767320115.143125:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320115.143129:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320115.143131:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982988928:0@lo:38:kworker.0 00000100:00100000:2.0:1767320115.143214:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989056 00000100:00100000:2.0:1767320115.143219:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989056 00000100:00100000:2.0:1767320115.143259:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 527 00000100:00100000:2.0:1767320115.143266:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982989056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320115.143320:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320115.143356:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982989056:12345-0@lo:38:kworker.0 Request processed in 90us (725us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320115.143362:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 527 00010000:00080000:2.0:1767320115.143393:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631909f80 x1853163982988544/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320131 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320115.143404:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982988544:0@lo:400:kworker.0 00000100:00080000:2.0:1767320115.143425:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320115.143489:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320115.143495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320115.143498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982989056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320115.144050:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989184 00000100:00100000:0.0:1767320115.144057:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989184 00000100:00100000:0.0:1767320115.144095:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010 00000100:00100000:0.0:1767320115.144101:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982989184:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320115.144113:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320115.144164:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982989184:12345-0@lo:38:kworker.0 Request processed in 63us (1094us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320115.144174:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010 00000100:00080000:1.0:1767320115.144316:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320115.144321:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320115.144324:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982989184:0@lo:38:kworker.0 00010000:00010000:1.0:1767320115.332901:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320116.357336:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320117.150167:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320117.150235:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320117.150499:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663232 00000100:00100000:2.0:1767320117.150514:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663232 00000100:00100000:2.0:1767320117.150570:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663360 00000100:00100000:2.0:1767320117.150574:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663360 00000100:00100000:2.0:1767320117.150633:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1522 00000100:00100000:2.0:1767320117.150646:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915663360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320117.150757:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915663360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 113us (520us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320117.150765:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1522 00000100:00100000:2.0:1767320117.150787:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2011 00000100:00100000:2.0:1767320117.150793:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915663232:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320117.150821:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320117.150851:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915663232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (702us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320117.150858:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2011 00000100:00080000:2.0:1767320117.150876:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320117.383869:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320118.404496:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320119.430891:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320119.748822:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320119.748860:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982989312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320119.748971:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320119.749564:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989312 00000100:00100000:1.0:1767320119.749580:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989312 00000100:00100000:1.0:1767320119.749687:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024 00000100:00100000:1.0:1767320119.749702:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982989312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320119.749755:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320119.749850:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982989312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 150us (878us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320119.749858:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024 00000100:00100000:1.0:1767320119.749940:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982989312:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320119.750025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320119.750433:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320119.750448:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982989440:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320119.750501:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320119.750594:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989440 00000100:00100000:0.0:1767320119.750601:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989440 00000100:00100000:0.0:1767320119.750664:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025 00000100:00100000:0.0:1767320119.750674:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982989440:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320119.750813:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:0.0:1767320119.750872:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320119.750932:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982989440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 260us (430us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320119.750940:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025 00000100:00100000:3.0:1767320119.751019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982989440:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320120.260858:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320120.260874:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320120.260941:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320120.260946:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320120.260951:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320120.260961:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5363 00000100:00080000:0.0:1767320120.260967:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320120.260975:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320120.261004:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b800 x1853163982989696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320120.261007:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982989568:0@lo:400:kworker.0 00000100:00080000:0.0:1767320120.261052:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320120.261056:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320120.261057:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320120.261062:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5363 00000100:00080000:0.0:1767320120.261065:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320120.261069:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320120.261083:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190ad80 x1853163982989952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320120.261093:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320120.261096:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320120.261097:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320120.261101:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5363 00000100:00080000:0.0:1767320120.261104:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320120.261108:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320120.261120:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631908380 x1853163982990080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320120.261130:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320120.261136:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320120.261153:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320120.261156:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320120.261160:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5363 00000100:00080000:0.0:1767320120.261163:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320120.261167:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320120.261181:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190aa00 x1853163982990208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320120.261208:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989568 00000100:00100000:0.0:1767320120.261223:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320120.261230:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989568 00000100:00100000:0.0:1767320120.261235:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982989824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320120.261290:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320120.261324:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989824 00000100:00100000:0.0:1767320120.261328:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989824 00000100:00100000:0.0:1767320120.261394:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1523 00000100:00100000:0.0:1767320120.261406:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982989824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320120.261473:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982989824:12345-0@lo:400:kworker.0 Request processed in 69us (201us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320120.261482:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1523 00000100:00100000:0.0:1767320120.261505:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1524 00000100:00100000:0.0:1767320120.261514:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982989568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320120.261534:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982989696:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.261555:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982989568:12345-0@lo:400:kworker.0 Request processed in 42us (428us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320120.261562:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1524 00000100:00100000:1.0:1767320120.261570:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320120.261581:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989696 00000100:00100000:0.0:1767320120.261586:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989696 00000100:00100000:1.0:1767320120.261592:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982989952:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.261611:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2012 00000100:00100000:0.0:1767320120.261618:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982989696:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320120.261622:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320120.261639:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982990080:0@lo:38:kworker.0 00010000:00020000:0.0:1767320120.261644:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320120.261663:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320120.261677:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982990208:0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.261689:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982989696:12345-0@lo:38:kworker.0 Request processed in 69us (118us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320120.261696:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320120.261699:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2012 00000100:00100000:0.0:1767320120.261714:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982989952 00000100:00100000:0.0:1767320120.261717:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982989952 00000100:00100000:0.0:1767320120.261727:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990208 00000100:00100000:0.0:1767320120.261729:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990208 00000100:00080000:1.0:1767320120.261733:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320120.261740:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013 00000100:00080000:1.0:1767320120.261741:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.261747:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982989952:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320120.261749:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982989696:0@lo:38:kworker.0 00010000:00020000:0.0:1767320120.261761:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320120.261787:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2014 00000100:00100000:1.0:1767320120.261794:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982990208:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320120.261798:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982989952:12345-0@lo:38:kworker.0 Request processed in 49us (176us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320120.261809:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013 00010000:00020000:1.0:1767320120.261809:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320120.261832:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320120.261848:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982990208:12345-0@lo:38:kworker.0 Request processed in 52us (150us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320120.261855:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2014 00010000:00080000:0.0:1767320120.261864:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631908700 x1853163982989824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320136 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320120.261877:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982989824:0@lo:400:kworker.0 00010000:00080000:1.0:1767320120.261894:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190a300 x1853163982989568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320136 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320120.261906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982989568:0@lo:400:kworker.0 00000100:00080000:0.0:1767320120.261908:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.261916:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.261920:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982989952:0@lo:38:kworker.0 00000100:00080000:0.0:1767320120.261931:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.261935:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.261939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982990208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320120.262407:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990080 00000100:00100000:3.0:1767320120.262415:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990080 00000100:00100000:3.0:1767320120.262451:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 528 00000100:00100000:3.0:1767320120.262457:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982990080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320120.262480:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320120.262528:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982990080:12345-0@lo:38:kworker.0 Request processed in 70us (864us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320120.262535:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 528 00000100:00080000:0.0:1767320120.262796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320120.262800:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320120.262804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982990080:0@lo:38:kworker.0 00010000:00010000:2.0:1767320120.452597:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320121.487559:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320122.255547:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320122.255591:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320122.255700:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663616 00000100:00100000:3.0:1767320122.255709:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663616 00000100:00100000:3.0:1767320122.255770:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663488 00000100:00100000:3.0:1767320122.255773:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663488 00000100:00100000:3.0:1767320122.255792:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1525 00000100:00100000:3.0:1767320122.255802:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915663616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320122.255863:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915663616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 63us (326us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320122.255868:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1525 00000100:00100000:3.0:1767320122.255882:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2015 00000100:00100000:3.0:1767320122.255885:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915663488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320122.255900:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320122.255916:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915663488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 30us (325us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320122.255920:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2015 00000100:00080000:2.0:1767320122.255976:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320122.501032:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320123.524662:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320124.550813:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320124.868589:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320124.868615:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982990336:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320124.868695:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320124.868795:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990336 00000100:00100000:0.0:1767320124.868805:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990336 00000100:00100000:0.0:1767320124.868874:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026 00000100:00100000:0.0:1767320124.868884:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982990336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320124.868915:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320124.868972:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982990336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320124.868980:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026 00000100:00080000:2.0:1767320124.869044:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320124.869124:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982990336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320124.870555:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320124.870583:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982990464:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320124.870638:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320124.870719:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990464 00000100:00100000:1.0:1767320124.870726:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990464 00000100:00100000:1.0:1767320124.870799:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1027 00000100:00100000:1.0:1767320124.870808:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982990464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320124.870834:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320124.870891:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982990464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (252us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320124.870898:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1027 00000100:00100000:3.0:1767320124.871037:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982990464:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320124.871309:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320125.380487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.380497:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320125.380570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.380575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.380579:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.380587:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.380593:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.380599:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320125.380620:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163982990720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.380657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.380661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.380663:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.380667:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.380670:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.380674:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320125.380687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0a80 x1853163982990976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.380697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320125.380699:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982990720:0@lo:38:kworker.0 00000100:00080000:3.0:1767320125.380701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.380705:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.380711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.380713:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.380717:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320125.380730:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1f80 x1853163982991104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320125.380739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320125.380743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320125.380745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320125.380748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5368 00000100:00080000:3.0:1767320125.380751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320125.380755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320125.380767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2a00 x1853163982991232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320125.380785:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320125.380812:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320125.380824:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982990848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320125.380830:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982990976:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.380866:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320125.380888:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990720 00000100:00100000:0.0:1767320125.381076:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990720 00000100:00100000:0.0:1767320125.381118:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990848 00000100:00100000:0.0:1767320125.381122:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990848 00000100:00100000:0.0:1767320125.381170:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1526 00000100:00100000:0.0:1767320125.381180:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982990848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320125.381196:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320125.381209:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982990592:0@lo:400:kworker.0 00000100:00100000:0.0:1767320125.381236:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982990848:12345-0@lo:400:kworker.0 Request processed in 57us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320125.381247:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1526 00000100:00100000:2.0:1767320125.381250:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320125.381780:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990592 00000100:00100000:0.0:1767320125.381786:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990592 00000100:00100000:0.0:1767320125.381809:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1527 00000100:00100000:0.0:1767320125.381815:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982990592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320125.381855:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982990592:12345-0@lo:400:kworker.0 Request processed in 40us (610us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320125.381861:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1527 00000100:00100000:0.0:1767320125.381874:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016 00000100:00100000:0.0:1767320125.381879:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982990720:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320125.381973:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320125.382006:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982990720:12345-0@lo:38:kworker.0 Request processed in 125us (1223us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320125.382011:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016 00010000:00080000:0.0:1767320125.382056:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321500 x1853163982990848/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320141 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320125.382068:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982990848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320125.382096:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320125.382129:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982991104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320125.382768:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320125.382808:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982991232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320125.382833:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320125.382862:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320125.382868:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320125.382872:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982990720:0@lo:38:kworker.0 00000100:00100000:3.0:1767320125.383211:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982990976 00000100:00100000:3.0:1767320125.383220:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982990976 00000100:00100000:3.0:1767320125.383246:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991232 00000100:00100000:3.0:1767320125.383249:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991232 00000100:00100000:3.0:1767320125.383259:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017 00000100:00100000:3.0:1767320125.383265:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982990976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320125.383295:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320125.383315:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2018 00000100:00100000:0.0:1767320125.383320:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982991232:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320125.383330:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320125.383340:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982990976:12345-0@lo:38:kworker.0 Request processed in 74us (1243us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320125.383352:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982991232:12345-0@lo:38:kworker.0 Request processed in 31us (519us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320125.384218:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017 00000100:00080000:3.0:1767320125.384289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320125.384331:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581f80 x1853163982990592/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320141 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320125.384345:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982990592:0@lo:400:kworker.0 00000100:00100000:3.0:1767320125.384401:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991104 00000100:00100000:3.0:1767320125.384407:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991104 00000100:00100000:3.0:1767320125.384433:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 529 00000100:00100000:3.0:1767320125.384438:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982991104:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320125.384452:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320125.385503:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982991104:12345-0@lo:38:kworker.0 Request processed in 1058us (2730us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320125.385523:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 529 00000100:00080000:3.0:1767320125.385625:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.385630:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.385634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982990976:0@lo:38:kworker.0 00000100:00080000:3.0:1767320125.385645:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.385648:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.385658:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982991104:0@lo:38:kworker.0 00000100:00080000:3.0:1767320125.385669:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320125.385673:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320125.385676:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982991232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320125.386709:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2018 00010000:00010000:2.0:1767320125.573931:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320126.596507:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320127.371504:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320127.371610:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663872 00000100:00100000:2.0:1767320127.371619:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663872 00000100:00100000:2.0:1767320127.371697:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1528 00000100:00100000:2.0:1767320127.371719:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915663872:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320127.371809:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915663872:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320127.371817:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1528 00000100:00080000:3.0:1767320127.371831:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320127.371960:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320127.372244:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915663744 00000100:00100000:2.0:1767320127.372249:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915663744 00000100:00100000:2.0:1767320127.372308:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2019 00000100:00100000:2.0:1767320127.372314:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915663744:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320127.372334:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320127.372378:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915663744:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 62us (419us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320127.372385:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2019 00010000:00010000:1.0:1767320127.620422:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320128.644560:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320129.668536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320129.988562:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982991360:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320129.988660:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320129.988810:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991360 00000100:00100000:0.0:1767320129.988819:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991360 00000100:00100000:0.0:1767320129.988876:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767320129.988885:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982991360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320129.988913:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320129.988976:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982991360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (316us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320129.988984:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767320129.989255:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320129.989327:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982991488:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320129.989356:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320129.989429:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991488 00000100:00100000:1.0:1767320129.989434:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991488 00000100:00100000:1.0:1767320129.989467:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029 00000100:00100000:1.0:1767320129.989474:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982991488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320129.989503:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320129.989536:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982991488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (179us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320129.989542:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029 00000100:00100000:3.0:1767320129.990070:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982991360:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320129.991505:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320129.991514:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320129.997628:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982991488:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320130.500637:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320130.500648:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320130.500705:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320130.500709:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320130.500713:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320130.500721:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5373 00000100:00080000:2.0:1767320130.500726:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320130.500731:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767320130.500733:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320130.500747:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982991616:0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.500754:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55880 x1853163982991744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320130.500798:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320130.500801:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320130.500802:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320130.500806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5373 00000100:00080000:2.0:1767320130.500808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320130.500811:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320130.500812:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320130.500823:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b800 x1853163982992000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320130.500831:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320130.500834:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320130.500836:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320130.500839:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5373 00000100:00080000:2.0:1767320130.500841:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320130.500844:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320130.500853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989500 x1853163982992128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320130.500875:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320130.500878:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991616 00000100:00080000:2.0:1767320130.500878:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320130.500880:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320130.500883:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5373 00000100:00100000:1.0:1767320130.500885:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991616 00000100:00080000:2.0:1767320130.500885:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320130.500888:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320130.500896:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989f80 x1853163982992512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320130.500927:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982991872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320130.500947:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1529 00000100:00100000:2.0:1767320130.500964:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320130.500965:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982991616:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.500983:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320130.500988:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982992256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.501011:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320130.501012:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982991616:12345-0@lo:400:kworker.0 Request processed in 48us (200us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.501019:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1529 00000100:00100000:1.0:1767320130.501034:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991872 00000100:00100000:1.0:1767320130.501037:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991872 00000100:00080000:2.0:1767320130.501039:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320130.501057:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1530 00000100:00100000:1.0:1767320130.501063:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982991872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320130.501069:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992256 00000100:00100000:2.0:1767320130.501072:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992256 00000100:00100000:1.0:1767320130.501096:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982991872:12345-0@lo:400:kworker.0 Request processed in 33us (132us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320130.501102:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1530 00000100:00100000:1.0:1767320130.501221:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982991744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320130.501247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320130.501268:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982992000:0@lo:38:kworker.0 00000100:00100000:3.0:1767320130.501309:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982991744 00000100:00100000:1.0:1767320130.501313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320130.501314:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982991744 00000100:00100000:1.0:1767320130.501331:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982992128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320130.501331:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992000 00000100:00100000:3.0:1767320130.501333:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992000 00000100:00100000:3.0:1767320130.501348:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2020 00000100:00100000:1.0:1767320130.501352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320130.501353:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982991744:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320130.501370:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982992512:0@lo:38:kworker.0 00010000:00020000:3.0:1767320130.501370:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.501388:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320130.501398:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982991744:12345-0@lo:38:kworker.0 Request processed in 45us (151us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320130.501405:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2020 00000100:00080000:1.0:1767320130.501412:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320130.501414:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992512 00000100:00080000:1.0:1767320130.501417:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320130.501417:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992512 00000100:00100000:1.0:1767320130.501420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982991744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320130.501426:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2021 00000100:00100000:3.0:1767320130.501430:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982992000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320130.501439:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.501441:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992128 00000100:00100000:1.0:1767320130.501444:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992128 00000100:00100000:1.0:1767320130.501463:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 530 00000100:00100000:3.0:1767320130.501464:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982992000:12345-0@lo:38:kworker.0 Request processed in 33us (151us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.501467:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982992128:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320130.501470:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2021 00010000:00020000:1.0:1767320130.501476:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320130.501480:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022 00000100:00100000:3.0:1767320130.501484:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982992512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320130.501491:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320130.501499:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982992128:12345-0@lo:38:kworker.0 Request processed in 31us (147us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320130.501505:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 530 00000100:00100000:3.0:1767320130.501508:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982992512:12345-0@lo:38:kworker.0 Request processed in 24us (120us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320130.501512:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022 00000100:00100000:1.0:1767320130.501521:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982992384:0@lo:400:kworker.0 00000100:00100000:1.0:1767320130.501540:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767320130.501643:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54380 x1853163982991872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320146 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320130.501653:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982991872:0@lo:400:kworker.0 00000100:00080000:0.0:1767320130.502593:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320130.502603:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320130.502606:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982992000:0@lo:38:kworker.0 00000100:00080000:0.0:1767320130.502617:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320130.502621:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320130.502624:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982992128:0@lo:38:kworker.0 00000100:00080000:0.0:1767320130.502633:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320130.502637:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320130.502640:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982992512:0@lo:38:kworker.0 00010000:00080000:1.0:1767320130.502927:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55500 x1853163982991616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320146 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320130.502940:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982991616:0@lo:400:kworker.0 00000100:00100000:0.0:1767320130.503082:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992384 00000100:00100000:0.0:1767320130.503088:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992384 00000100:00100000:0.0:1767320130.503139:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953 00000100:00100000:0.0:1767320130.503148:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982992256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320130.503211:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982992256:12345-0@lo:400:kworker.0 Request processed in 64us (2198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320130.503217:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953 00000100:00100000:0.0:1767320130.503232:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 954 00000100:00100000:0.0:1767320130.503238:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982992384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320130.503267:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982992384:12345-0@lo:400:kworker.0 Request processed in 29us (1726us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320130.503292:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 954 00000100:00100000:1.0:1767320130.503403:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982992384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320130.503530:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982992256:0@lo:400:kworker.0 00010000:00010000:1.0:1767320130.694420:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320131.717466:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320132.494748:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320132.494861:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664000 00000100:00100000:1.0:1767320132.494872:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664000 00000100:00100000:1.0:1767320132.494935:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2023 00000100:00100000:1.0:1767320132.494944:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915664000:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320132.494968:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320132.495039:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915664000:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (303us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320132.495046:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2023 00000100:00100000:0.0:1767320132.497419:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320132.500401:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664128 00000100:00100000:3.0:1767320132.500416:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664128 00000100:00100000:3.0:1767320132.500530:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1531 00000100:00100000:3.0:1767320132.500541:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915664128:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320132.500635:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915664128:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 96us (3225us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320132.500645:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1531 00000100:00080000:2.0:1767320132.501007:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320132.740513:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320133.766142:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320134.790113:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320135.109348:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320135.109368:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982992640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320135.109445:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320135.109700:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992640 00000100:00100000:2.0:1767320135.109709:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992640 00000100:00100000:2.0:1767320135.109784:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1030 00000100:00100000:2.0:1767320135.109794:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982992640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320135.109831:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320135.109917:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982992640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 123us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320135.109923:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1030 00000100:00100000:2.0:1767320135.109971:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982992640:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320135.110034:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320135.114576:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982992768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320135.114651:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320135.114821:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992768 00000100:00100000:1.0:1767320135.114832:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992768 00000100:00100000:1.0:1767320135.114911:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031 00000100:00100000:1.0:1767320135.114921:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982992768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320135.114955:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320135.115050:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982992768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 130us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320135.115059:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031 00000100:00080000:3.0:1767320135.115149:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320135.117425:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982992768:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320135.620401:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320135.620412:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320135.620505:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320135.620509:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320135.620513:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320135.620521:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5379 00000100:00080000:2.0:1767320135.620527:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320135.620534:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320135.620936:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982992896:0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.621016:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320135.621161:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989180 x1853163982993024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320135.621212:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320135.621216:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320135.621218:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320135.621222:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5379 00000100:00080000:2.0:1767320135.621224:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320135.621228:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320135.621241:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198ad80 x1853163982993280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320135.621249:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320135.621253:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320135.621255:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320135.621258:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5379 00000100:00080000:2.0:1767320135.621260:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320135.621264:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320135.621294:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988a80 x1853163982993408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320135.621298:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982992896 00000100:00100000:1.0:1767320135.621306:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982992896 00000100:00080000:2.0:1767320135.621318:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320135.621321:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320135.621323:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320135.621326:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5379 00000100:00080000:2.0:1767320135.621328:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320135.621331:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320135.621339:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988e00 x1853163982993792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320135.621357:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1532 00000100:00100000:2.0:1767320135.621359:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982993536:0@lo:400:kworker.0 00000100:00100000:1.0:1767320135.621366:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982992896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320135.621388:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320135.621407:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320135.621412:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982993152:0@lo:400:kworker.0 00000100:00100000:1.0:1767320135.621413:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982992896:12345-0@lo:400:kworker.0 Request processed in 49us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320135.621421:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1532 00000100:00100000:2.0:1767320135.621429:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320135.621479:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982993024:0@lo:38:kworker.0 00000100:00100000:3.0:1767320135.621516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320135.621761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982993664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320135.621792:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320135.621823:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320135.621852:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198a300 x1853163982992896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320151 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320135.621879:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982992896:0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.628298:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982993280:0@lo:38:kworker.0 00000100:00100000:3.0:1767320135.628365:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320135.628385:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982993408:0@lo:38:kworker.0 00000100:00100000:3.0:1767320135.628409:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320135.628431:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982993792:0@lo:38:kworker.0 00000100:00100000:3.0:1767320135.628452:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320135.628716:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993024 00000100:00100000:3.0:1767320135.628725:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993024 00000100:00100000:3.0:1767320135.628750:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993280 00000100:00100000:3.0:1767320135.628753:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993280 00000100:00100000:3.0:1767320135.628761:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993792 00000100:00100000:3.0:1767320135.628762:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993792 00000100:00100000:3.0:1767320135.628775:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1767320135.628781:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993024:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320135.628802:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320135.628835:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993152 00000100:00100000:1.0:1767320135.628841:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993152 00000100:00100000:3.0:1767320135.628846:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993024:12345-0@lo:38:kworker.0 Request processed in 65us (7329us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320135.628864:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1767320135.628882:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025 00000100:00100000:3.0:1767320135.628886:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993280:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.628890:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1533 00000100:00100000:1.0:1767320135.628896:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982993152:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767320135.628896:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320135.628918:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993280:12345-0@lo:38:kworker.0 Request processed in 31us (554us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320135.628924:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025 00000100:00100000:3.0:1767320135.628936:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2026 00000100:00100000:3.0:1767320135.628939:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993792:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.628940:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163982993152:12345-0@lo:400:kworker.0 Request processed in 45us (7509us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320135.628946:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1533 00010000:00020000:3.0:1767320135.628948:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320135.628965:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320135.628967:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982993792:12345-0@lo:38:kworker.0 Request processed in 27us (514us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320135.628972:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2026 00000100:00100000:3.0:1767320135.628991:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993408 00000100:00100000:3.0:1767320135.628993:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993408 00010000:00080000:1.0:1767320135.628999:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198b480 x1853163982993152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320151 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320135.629011:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 531 00000100:00100000:1.0:1767320135.629013:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163982993152:0@lo:400:kworker.0 00000100:00100000:3.0:1767320135.629015:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982993408:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320135.629023:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320135.629053:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993536 00000100:00100000:1.0:1767320135.629056:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993536 00000100:00100000:1.0:1767320135.629080:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993664 00000100:00100000:1.0:1767320135.629082:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993664 00000100:00100000:1.0:1767320135.629100:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 955 00000100:00100000:1.0:1767320135.629106:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982993536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320135.629139:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982993536:12345-0@lo:400:kworker.0 Request processed in 33us (7750us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320135.629142:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982993408:12345-0@lo:38:kworker.0 Request processed in 29us (635us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320135.629145:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 955 00000100:00100000:3.0:1767320135.629149:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 531 00000100:00100000:1.0:1767320135.629157:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956 00000100:00100000:1.0:1767320135.629162:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982993664:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320135.629175:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.629180:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.629183:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982993024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320135.629191:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982993664:12345-0@lo:400:kworker.0 Request processed in 30us (7399us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320135.629196:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956 00000100:00100000:1.0:1767320135.629220:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982993536:0@lo:400:kworker.0 00000100:00080000:3.0:1767320135.629231:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.629234:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.629237:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982993280:0@lo:38:kworker.0 00000100:00080000:3.0:1767320135.629245:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.629248:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.629251:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982993408:0@lo:38:kworker.0 00000100:00080000:3.0:1767320135.629258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320135.629260:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320135.629263:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982993792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320135.630193:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982993664:0@lo:400:kworker.0 00010000:00010000:2.0:1767320135.812731:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320136.836551:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320137.630544:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767320137.630589:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915664256 offset 0 length 520: 4 00000100:00100000:0.0:1767320137.630631:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320137.630724:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664512 00000100:00100000:3.0:1767320137.630735:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664512 00000100:00100000:3.0:1767320137.630892:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664384 00000100:00100000:3.0:1767320137.630896:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664384 00000100:00100000:3.0:1767320137.630928:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1534 00000100:00100000:3.0:1767320137.630938:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915664512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320137.631029:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915664512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (495us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320137.631038:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1534 00000100:00100000:0.0:1767320137.631056:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2027 00000100:00100000:0.0:1767320137.631064:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915664384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320137.631085:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320137.631113:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915664384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (483us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320137.631584:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320137.636810:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2027 00010000:00010000:2.0:1767320137.860465:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320138.885233:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320139.909031:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320140.228441:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320140.228461:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982993920:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320140.228540:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320140.228606:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982993920 00000100:00100000:2.0:1767320140.228613:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982993920 00000100:00100000:2.0:1767320140.228663:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032 00000100:00100000:2.0:1767320140.228671:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982993920:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320140.228679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:2.0:1767320140.228699:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320140.228756:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982993920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (215us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320140.228762:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032 00000100:00100000:0.0:1767320140.229033:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982993920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320140.229443:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320140.229455:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982994048:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320140.229488:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320140.235500:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994048 00000100:00100000:3.0:1767320140.235516:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994048 00000100:00100000:3.0:1767320140.235592:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1033 00000100:00100000:3.0:1767320140.235602:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982994048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320140.235637:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320140.235720:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982994048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 120us (6230us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320140.235728:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1033 00000100:00100000:3.0:1767320140.235800:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982994048:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320140.236184:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320140.744044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320140.744054:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320140.744114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320140.744118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320140.744121:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320140.744129:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5384 00000100:00080000:3.0:1767320140.744134:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320140.744135:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320115/real 1767320115] req@ffff934631909180 x1853163982988416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320140 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320140.744141:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320140.744159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1180 x1853163982994304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320140.744164:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631909180 x1853163982988416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320140 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320140.744183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320140.744185:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982994304:0@lo:38:kworker.0 00000100:00080000:3.0:1767320140.744186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320140.744188:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320140.744191:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5384 00000100:00080000:3.0:1767320140.744193:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320140.744197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320140.744205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1500 x1853163982994560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320140.744212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320140.744214:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320140.744216:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320140.744219:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5384 00000100:00080000:3.0:1767320140.744221:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320140.744224:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320140.744233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0700 x1853163982994688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320140.744245:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320140.744250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320140.744253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320140.744254:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320140.744257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5384 00000100:00080000:3.0:1767320140.744259:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320140.744262:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320140.744285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163982995072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320140.744311:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982994944:0@lo:400:kworker.0 00000100:00080000:1.0:1767320140.744317:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320140.744330:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320140.744334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982988416:0@lo:250:kworker.0 00000100:00100000:3.0:1767320140.744341:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320140.744354:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982994560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320140.744361:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320140.744365:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982994176:0@lo:400:kworker.0 00000100:00100000:1.0:1767320140.744375:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320140.744396:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320140.744400:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982994688:0@lo:38:kworker.0 00000100:00100000:1.0:1767320140.744421:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320140.744425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320140.744428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320140.744429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320140.744433:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5363 00000100:00080000:3.0:1767320140.744436:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320140.744440:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320140.744453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581f80 x1853163982995200/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320140.744454:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982995072:0@lo:38:kworker.0 00000100:00100000:1.0:1767320140.744473:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320140.744477:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994688 00000100:00100000:3.0:1767320140.744483:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994688 00000100:00100000:1.0:1767320140.744501:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982995200:0@lo:250:kworker.0 00000400:00000100:1.0:1767320140.744518:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982995200 offset 0 length 520: 4 00000100:00100000:3.0:1767320140.744518:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 532 00000100:00100000:3.0:1767320140.744525:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982994688:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320140.744543:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994304 00010000:00020000:3.0:1767320140.744543:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320140.744546:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994304 00000100:00100000:1.0:1767320140.744564:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994560 00000100:00100000:1.0:1767320140.744567:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994560 00000100:00100000:1.0:1767320140.744574:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994176 00000100:00100000:1.0:1767320140.744577:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994176 00000100:00100000:3.0:1767320140.744579:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982994688:12345-0@lo:38:kworker.0 Request processed in 54us (157us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320140.744585:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 532 00000100:00100000:1.0:1767320140.744601:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995072 00000100:00100000:1.0:1767320140.744604:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995072 00000100:00100000:1.0:1767320140.744613:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1535 00000100:00100000:1.0:1767320140.744619:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982994176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320140.744660:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163982994176:12345-0@lo:400:kworker.0 Request processed in 41us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320140.744720:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994944 00000100:00100000:2.0:1767320140.744724:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994944 00000100:00100000:2.0:1767320140.744754:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957 00000100:00100000:2.0:1767320140.744813:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982994944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320140.744847:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982994944:12345-0@lo:400:kworker.0 Request processed in 88us (505us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320140.744853:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957 00000100:00100000:3.0:1767320140.748047:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028 00000100:00100000:3.0:1767320140.748061:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982994304:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320140.748085:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320140.748166:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982994304:12345-0@lo:38:kworker.0 Request processed in 107us (3920us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320140.748173:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028 00000100:00100000:3.0:1767320140.748203:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029 00000100:00100000:3.0:1767320140.748207:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982994560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320140.748216:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320140.748239:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982994560:12345-0@lo:38:kworker.0 Request processed in 31us (3862us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320140.748245:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029 00000100:00100000:3.0:1767320140.748256:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2030 00000100:00100000:3.0:1767320140.748260:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982995072:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320140.748285:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320140.748312:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982995072:12345-0@lo:38:kworker.0 Request processed in 50us (3837us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320140.748320:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2030 00000100:00080000:3.0:1767320140.748389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320140.748426:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982994816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748488:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320140.748523:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320140.748529:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982994432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748551:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320140.748578:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994432 00000100:00100000:3.0:1767320140.748584:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994432 00000100:00100000:3.0:1767320140.748626:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1536 00000100:00100000:3.0:1767320140.748631:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982994432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748668:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982994432:12345-0@lo:400:kworker.0 Request processed in 36us (116us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320140.748673:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1536 00000100:00100000:3.0:1767320140.748691:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982994816 00000100:00100000:3.0:1767320140.748694:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982994816 00000100:00100000:3.0:1767320140.748722:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 958 00000100:00100000:3.0:1767320140.748727:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982994816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748753:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982994816:12345-0@lo:400:kworker.0 Request processed in 26us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320140.748758:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 958 00000100:00080000:3.0:1767320140.748784:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320140.748809:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a2300 x1853163982994432/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320156 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320140.748824:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982994432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748833:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982994816:0@lo:400:kworker.0 00000100:00080000:3.0:1767320140.748888:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320140.748894:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320140.748898:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982994560:0@lo:38:kworker.0 00000100:00080000:3.0:1767320140.748907:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320140.748910:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320140.748913:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982994688:0@lo:38:kworker.0 00000100:00080000:3.0:1767320140.748921:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320140.748924:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320140.748926:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982995072:0@lo:38:kworker.0 00000100:00080000:3.0:1767320140.748934:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320140.748937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320140.748939:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982994304:0@lo:38:kworker.0 00010000:00080000:3.0:1767320140.748976:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3800 x1853163982994176/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320156 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320140.748985:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163982994176:0@lo:400:kworker.0 00000100:00100000:3.0:1767320140.748995:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982994944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320140.751879:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1535 00010000:00010000:2.0:1767320140.933140:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320141.956613:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320142.732979:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320142.733020:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320142.733043:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664768 00000100:00100000:2.0:1767320142.733057:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664768 00000100:00100000:0.0:1767320142.733119:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664640 00000100:00100000:0.0:1767320142.733124:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664640 00000100:00100000:0.0:1767320142.733220:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2031 00000100:00100000:0.0:1767320142.733228:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915664640:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320142.733252:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320142.733369:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915664640:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 140us (347us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320142.733377:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2031 00000100:00080000:3.0:1767320142.733608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320142.733829:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1537 00000100:00100000:1.0:1767320142.733840:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915664768:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320142.733948:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915664768:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 110us (979us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320142.733956:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1537 00010000:00010000:1.0:1767320142.980627:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320144.004543:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320145.029059:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320145.349739:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320145.349768:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982995328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320145.349870:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.350199:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995328 00000100:00100000:1.0:1767320145.350209:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995328 00000100:00100000:1.0:1767320145.350318:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034 00000100:00100000:1.0:1767320145.350329:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346325d1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982995328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320145.350361:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320145.350414:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346325d1c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982995328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (544us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320145.350422:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034 00000100:00100000:2.0:1767320145.352081:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320145.352097:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982995456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320145.352157:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320145.352251:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320145.352322:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982995328:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320145.353154:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995456 00000100:00100000:0.0:1767320145.353164:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995456 00000100:00100000:0.0:1767320145.353232:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035 00000100:00100000:0.0:1767320145.353242:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982995456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320145.353290:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320145.353356:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163982995456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 116us (1198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320145.353363:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035 00000100:00100000:2.0:1767320145.353501:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163982995456:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320145.353622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320145.860722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320145.860734:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320145.860977:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982995584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320145.861050:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320145.861138:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995584 00000100:00100000:3.0:1767320145.861147:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995584 00000100:00100000:3.0:1767320145.861223:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1538 00000100:00100000:3.0:1767320145.861233:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982995584:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320145.861344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320145.861350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320145.861353:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320145.861359:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5389 00000100:00080000:0.0:1767320145.861362:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320145.861368:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320145.861394:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4e00 x1853163982995712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320145.861431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320145.861434:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320145.861436:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320145.861440:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5389 00000100:00080000:0.0:1767320145.861442:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320145.861446:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320145.861459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5180 x1853163982995968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320145.861468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320145.861470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320145.861472:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320145.861475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5389 00000100:00080000:0.0:1767320145.861477:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320145.861481:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320145.861492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5880 x1853163982996096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320145.861548:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320145.861578:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982996352:0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.861617:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.861676:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982995712:0@lo:38:kworker.0 00000100:00100000:1.0:1767320145.861702:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.861718:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982995968:0@lo:38:kworker.0 00000100:00100000:1.0:1767320145.861737:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.861751:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982996096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320145.861772:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.861823:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995712 00000100:00100000:1.0:1767320145.861828:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995712 00000100:00100000:1.0:1767320145.861851:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995968 00000100:00100000:1.0:1767320145.861854:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995968 00000100:00100000:1.0:1767320145.861870:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2032 00000100:00100000:1.0:1767320145.861876:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982995712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320145.861896:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320145.861932:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982995712:12345-0@lo:38:kworker.0 Request processed in 55us (229us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320145.861939:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2032 00000100:00100000:1.0:1767320145.861956:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2033 00000100:00100000:1.0:1767320145.861960:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982995968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320145.861968:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320145.861992:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982995968:12345-0@lo:38:kworker.0 Request processed in 30us (254us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320145.861997:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2033 00000100:00100000:1.0:1767320145.862015:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996096 00000100:00100000:1.0:1767320145.862018:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996096 00000100:00100000:1.0:1767320145.862039:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 533 00000100:00100000:1.0:1767320145.862043:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982996096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320145.862052:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320145.862076:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982996096:12345-0@lo:38:kworker.0 Request processed in 31us (302us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320145.862081:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 533 00000100:00080000:1.0:1767320145.862123:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320145.862128:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320145.862131:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982995712:0@lo:38:kworker.0 00000100:00080000:1.0:1767320145.862140:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320145.862143:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320145.862146:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982995968:0@lo:38:kworker.0 00000100:00080000:1.0:1767320145.862153:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320145.862156:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320145.862158:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982996096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320145.862198:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996352 00000100:00100000:1.0:1767320145.862202:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996352 00000100:00100000:1.0:1767320145.862230:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959 00000100:00100000:1.0:1767320145.862237:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982996352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862302:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982996352:12345-0@lo:400:kworker.0 Request processed in 66us (684us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320145.862308:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959 00000100:00100000:1.0:1767320145.862327:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982995840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862350:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.862369:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320145.862375:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982996224:0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862395:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320145.862417:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982995840 00000100:00100000:1.0:1767320145.862421:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982995840 00000100:00100000:1.0:1767320145.862445:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1539 00000100:00100000:1.0:1767320145.862451:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982995840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862484:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163982995840:12345-0@lo:400:kworker.0 Request processed in 32us (132us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320145.862490:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1539 00000100:00080000:1.0:1767320145.862505:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320145.862524:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982996352:0@lo:400:kworker.0 00010000:00080000:1.0:1767320145.862546:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7b80 x1853163982995840/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320161 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320145.862555:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163982995840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862568:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996224 00000100:00100000:1.0:1767320145.862571:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996224 00000100:00100000:1.0:1767320145.862593:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960 00000100:00100000:1.0:1767320145.862599:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982996224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320145.862628:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163982996224:12345-0@lo:400:kworker.0 Request processed in 30us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320145.862635:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960 00000100:00100000:1.0:1767320145.862659:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163982996224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320145.868133:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982995584:12345-0@lo:400:kworker.0 Request processed in 6889us (7068us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320145.868151:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1538 00000100:00080000:0.0:1767320145.868258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320145.868264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320145.868282:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320145.868288:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5389 00000100:00080000:0.0:1767320145.868293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320145.868298:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320145.868340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5500 x1853163982996480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320145.869062:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982996480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320145.869135:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320145.869246:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996480 00000100:00100000:3.0:1767320145.869256:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996480 00000100:00100000:3.0:1767320145.869328:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034 00000100:00100000:3.0:1767320145.869334:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982996480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320145.869367:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320145.869415:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982996480:12345-0@lo:38:kworker.0 Request processed in 80us (280us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320145.869423:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034 00000100:00080000:0.0:1767320145.869580:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320145.869586:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320145.869590:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982996480:0@lo:38:kworker.0 00010000:00080000:2.0:1767320145.870326:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5c00 x1853163982995584/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320161 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320145.870341:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982995584:0@lo:400:kworker.0 00010000:00010000:1.0:1767320146.052675:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320147.079237:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320147.852552:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320147.852597:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320147.852644:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665024 00000100:00100000:1.0:1767320147.852656:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665024 00000100:00100000:1.0:1767320147.852744:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915664896 00000100:00100000:1.0:1767320147.852748:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915664896 00000100:00100000:1.0:1767320147.852771:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1540 00000100:00100000:1.0:1767320147.852783:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915665024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320147.852814:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2035 00000100:00100000:0.0:1767320147.852822:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915664896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320147.852844:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320147.852866:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915665024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320147.852875:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1540 00000100:00100000:0.0:1767320147.852883:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915664896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (284us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320147.852892:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2035 00000100:00080000:3.0:1767320147.853153:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320148.102826:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320149.126666:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320150.150918:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320150.468572:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320150.468626:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982996608:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320150.468704:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320150.468795:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996608 00000100:00100000:2.0:1767320150.468804:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996608 00000100:00080000:3.0:1767320150.468867:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320150.468871:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036 00000100:00100000:2.0:1767320150.468882:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982996608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320150.468910:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320150.468962:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982996608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320150.468969:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036 00000100:00100000:3.0:1767320150.469037:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982996608:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320150.469180:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320150.469190:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982996736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320150.469218:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320150.469342:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996736 00000100:00100000:0.0:1767320150.469346:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996736 00000100:00100000:0.0:1767320150.469386:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037 00000100:00100000:0.0:1767320150.469393:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982996736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320150.469409:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320150.469448:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982996736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (229us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320150.469454:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037 00000100:00100000:1.0:1767320150.469551:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982996736:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320150.469621:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320150.980440:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320150.980457:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320150.980564:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320150.980578:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320150.980582:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320150.980593:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320150.980601:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320150.980610:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320150.980637:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0a80 x1853163982996992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320150.980685:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320150.980689:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320150.980692:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320150.980696:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320150.980698:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320150.980702:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320150.980715:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163982997248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320150.980726:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320150.980729:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320150.980732:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320150.980737:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320150.980741:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320150.980748:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982996992:0@lo:38:kworker.0 00000100:00080000:3.0:1767320150.980748:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320150.980765:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3100 x1853163982997376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320150.980780:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320150.980784:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320150.980787:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320150.980791:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5394 00000100:00080000:3.0:1767320150.980794:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320150.980799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320150.980813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2d80 x1853163982997504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320150.980841:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320150.980856:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982997120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320150.980889:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982997248:0@lo:38:kworker.0 00000100:00100000:2.0:1767320150.980995:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982996864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320150.981115:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996992 00000100:00100000:0.0:1767320150.981125:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996992 00000100:00100000:0.0:1767320150.981178:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2036 00000100:00100000:0.0:1767320150.981188:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982996992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320150.981216:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320150.981268:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320150.981381:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997120 00000100:00100000:3.0:1767320150.981385:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997120 00000100:00100000:3.0:1767320150.981434:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1541 00000100:00100000:3.0:1767320150.981443:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982997120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320150.981501:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982997120:12345-0@lo:400:kworker.0 Request processed in 60us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320150.981508:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1541 00000100:00080000:3.0:1767320150.981534:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320150.981589:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3b80 x1853163982997120/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320166 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320150.981603:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982997120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320150.981861:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320150.981924:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982997376:0@lo:38:kworker.0 00000100:00100000:3.0:1767320150.982077:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997248 00000100:00100000:3.0:1767320150.982086:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997248 00000100:00100000:3.0:1767320150.982124:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037 00000100:00100000:3.0:1767320150.982131:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982997248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320150.982151:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320150.982611:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982997248:12345-0@lo:38:kworker.0 Request processed in 477us (751us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320150.982619:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037 00000100:00100000:1.0:1767320150.982800:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320150.982839:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982997504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320150.982866:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320150.985831:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320150.985845:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320150.985852:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982997248:0@lo:38:kworker.0 00000100:00100000:1.0:1767320150.985914:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997376 00000100:00100000:1.0:1767320150.985922:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997376 00000100:00100000:1.0:1767320150.985971:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 534 00000100:00100000:1.0:1767320150.985978:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982997376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320150.986000:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320150.986062:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982997376:12345-0@lo:38:kworker.0 Request processed in 82us (3261us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320150.986069:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 534 00000100:00080000:1.0:1767320150.986113:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320150.986118:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320150.986121:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982997376:0@lo:38:kworker.0 00000100:00100000:2.0:1767320150.986175:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320150.986381:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997504 00000100:00100000:1.0:1767320150.986388:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997504 00000100:00100000:1.0:1767320150.986429:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982996864 00000100:00100000:1.0:1767320150.986432:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982996864 00000100:00100000:1.0:1767320150.986483:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1542 00000100:00100000:1.0:1767320150.986491:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982996864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320150.986553:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982996864:12345-0@lo:400:kworker.0 Request processed in 64us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320150.986562:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1542 00000100:00100000:1.0:1767320150.986594:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2038 00000100:00100000:1.0:1767320150.986602:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982997504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320150.986620:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320150.986657:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982997504:12345-0@lo:38:kworker.0 Request processed in 53us (3789us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320150.986664:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2038 00000100:00080000:1.0:1767320150.986684:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320150.986714:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320150.986719:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320150.986723:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982997504:0@lo:38:kworker.0 00010000:00080000:2.0:1767320150.987145:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2680 x1853163982996864/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320166 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320150.987160:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982996864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320150.987321:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982996992:12345-0@lo:38:kworker.0 Request processed in 6131us (6484us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320150.987330:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2036 00000100:00080000:1.0:1767320150.988136:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320150.988143:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320150.988147:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982996992:0@lo:38:kworker.0 00010000:00010000:2.0:1767320151.172658:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320152.196502:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320152.973484:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320152.973554:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320152.973676:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665280 00000100:00100000:3.0:1767320152.973691:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665280 00000100:00100000:3.0:1767320152.973791:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665152 00000100:00100000:3.0:1767320152.973795:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665152 00000100:00100000:3.0:1767320152.973826:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1543 00000100:00100000:3.0:1767320152.973852:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915665280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320152.973869:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2039 00000100:00100000:0.0:1767320152.973878:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915665152:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320152.973905:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320152.973979:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320152.974654:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915665152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 769us (1093us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320152.974655:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915665280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 811us (1180us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320152.974672:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2039 00000100:00100000:3.0:1767320152.975118:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1543 00010000:00010000:2.0:1767320153.220999:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320154.244598:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320155.272587:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320155.588532:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320155.588583:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982997632:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320155.588694:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320155.588890:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997632 00000100:00100000:2.0:1767320155.588903:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997632 00000100:00080000:1.0:1767320155.588997:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320155.589002:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038 00000100:00100000:2.0:1767320155.589021:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982997632:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320155.589029:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320155.589058:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982997760:0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320155.589073:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320155.589149:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320155.589172:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982997632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 152us (475us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320155.589185:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038 00000100:00100000:2.0:1767320155.589208:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997760 00000100:00100000:2.0:1767320155.589214:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997760 00000100:00100000:2.0:1767320155.589264:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1039 00000100:00100000:2.0:1767320155.589310:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982997760:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320155.589315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767320155.589348:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320155.589400:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982997760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 128us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320155.589407:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1039 00000100:00100000:1.0:1767320155.589654:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982997760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320155.590186:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982997632:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320156.103742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320156.103758:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320156.104500:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982997888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320156.104605:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320156.104837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320156.104842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320156.104848:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320156.104856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5399 00000100:00080000:0.0:1767320156.104860:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320156.104869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320156.104902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6680 x1853163982998016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320156.104975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320156.104979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320156.104982:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320156.104985:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982998016:0@lo:38:kworker.0 00000100:00080000:0.0:1767320156.104988:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5399 00000100:00080000:0.0:1767320156.104994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320156.104999:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320156.105013:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4a80 x1853163982998272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320156.105024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320156.105028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320156.105030:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320156.105034:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5399 00000100:00080000:0.0:1767320156.105037:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320156.105042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320156.105045:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320156.105058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7100 x1853163982998400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320156.105407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320156.105411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320156.105413:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320156.105417:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5399 00000100:00080000:0.0:1767320156.105419:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320156.105423:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320156.105443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6a00 x1853163982998528/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320156.105491:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320156.105506:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982998144:0@lo:400:kworker.0 00000100:00100000:1.0:1767320156.108263:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320156.108288:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982998272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.111957:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320156.113282:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982997888 00000100:00100000:0.0:1767320156.113304:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982997888 00000100:00100000:2.0:1767320156.114726:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998016 00000100:00100000:2.0:1767320156.114744:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998016 00000100:00100000:2.0:1767320156.114804:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998144 00000100:00100000:2.0:1767320156.114808:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998144 00000100:00100000:2.0:1767320156.114844:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1544 00000100:00100000:2.0:1767320156.114856:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982997888:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.115339:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982997888:12345-0@lo:400:kworker.0 Request processed in 482us (10732us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320156.115350:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1544 00000100:00100000:2.0:1767320156.115376:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1545 00000100:00100000:2.0:1767320156.115385:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982998144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320156.115427:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163982998144:12345-0@lo:400:kworker.0 Request processed in 43us (7176us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320156.115433:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1545 00000100:00100000:2.0:1767320156.115447:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040 00000100:00100000:2.0:1767320156.115452:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982998016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320156.115486:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320156.115519:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163982998016:12345-0@lo:38:kworker.0 Request processed in 66us (10481us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320156.115525:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040 00010000:00080000:2.0:1767320156.115603:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5f80 x1853163982997888/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320172 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320156.115621:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982997888:0@lo:400:kworker.0 00010000:00080000:2.0:1767320156.115667:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7800 x1853163982998144/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320172 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320156.115678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163982998144:0@lo:400:kworker.0 00000100:00080000:2.0:1767320156.115726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320156.115957:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982998400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.116006:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320156.116037:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998400 00000100:00100000:1.0:1767320156.116042:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998400 00000100:00100000:1.0:1767320156.116068:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 535 00000100:00100000:1.0:1767320156.116073:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982998400:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320156.116084:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320156.116111:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982998400:12345-0@lo:38:kworker.0 Request processed in 37us (106us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320156.116117:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 535 00000100:00100000:1.0:1767320156.116136:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998272 00000100:00100000:1.0:1767320156.116139:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998272 00000100:00100000:1.0:1767320156.116159:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041 00000100:00100000:1.0:1767320156.116165:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982998272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320156.116175:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320156.116206:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163982998272:12345-0@lo:38:kworker.0 Request processed in 40us (4278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320156.116212:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041 00000100:00080000:1.0:1767320156.116249:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.116254:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.116257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982998400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.116288:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982998528:0@lo:38:kworker.0 00000100:00100000:1.0:1767320156.116315:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320156.116336:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998528 00000100:00100000:1.0:1767320156.116340:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998528 00000100:00100000:1.0:1767320156.116357:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2042 00000100:00100000:1.0:1767320156.116361:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982998528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320156.116371:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320156.116397:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163982998528:12345-0@lo:38:kworker.0 Request processed in 34us (81us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320156.116403:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2042 00000100:00080000:1.0:1767320156.116427:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.116431:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.116434:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982998528:0@lo:38:kworker.0 00000100:00080000:1.0:1767320156.116446:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.116451:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.116454:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982998016:0@lo:38:kworker.0 00000100:00080000:1.0:1767320156.116493:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320156.116497:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320156.116501:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982998272:0@lo:38:kworker.0 00010000:00010000:2.0:1767320156.292874:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320157.317712:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320158.094920:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320158.094975:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320158.095176:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665536 00000100:00100000:3.0:1767320158.095187:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665536 00000100:00100000:3.0:1767320158.095304:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665408 00000100:00100000:3.0:1767320158.095309:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665408 00000100:00100000:3.0:1767320158.095330:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1546 00000100:00100000:3.0:1767320158.095340:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915665536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320158.095421:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915665536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320158.095430:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1546 00000100:00100000:3.0:1767320158.095458:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2043 00000100:00100000:3.0:1767320158.095467:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915665408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320158.095489:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320158.095514:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915665408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (540us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320158.095521:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2043 00000100:00080000:2.0:1767320158.095814:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320158.340525:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320159.364473:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320160.389292:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320160.708627:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320160.708663:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982998656:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320160.708732:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320160.708892:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998656 00000100:00100000:1.0:1767320160.708900:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998656 00000100:00100000:1.0:1767320160.709023:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040 00000100:00100000:1.0:1767320160.709034:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982998656:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320160.709043:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767320160.709063:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320160.709119:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163982998656:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320160.709127:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040 00000100:00100000:3.0:1767320160.709199:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163982998656:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320160.709315:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320160.709325:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982998784:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320160.709353:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320160.710116:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998784 00000100:00100000:0.0:1767320160.710126:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998784 00000100:00100000:0.0:1767320160.710169:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041 00000100:00100000:0.0:1767320160.710177:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982998784:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320160.710199:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320160.710241:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163982998784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (887us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320160.710247:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041 00000100:00080000:2.0:1767320160.710304:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320160.710346:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163982998784:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320161.220436:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320161.220447:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320161.220525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320161.220530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320161.220533:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320161.220548:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5404 00000100:00080000:0.0:1767320161.220552:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320161.220560:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320161.220580:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed6680 x1853163982999040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320161.220629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320161.220632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320161.220634:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320161.220638:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5404 00000100:00080000:0.0:1767320161.220640:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320161.220644:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320161.220655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7100 x1853163982999296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320161.220664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320161.220667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320161.220669:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320161.220672:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5404 00000100:00080000:0.0:1767320161.220674:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320161.220677:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320161.220687:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7800 x1853163982999424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320161.220696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320161.220699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320161.220701:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320161.220704:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5404 00000100:00080000:0.0:1767320161.220706:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320161.220709:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320161.220717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5f80 x1853163982999552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320161.220742:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982999168:0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.220803:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320161.220875:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999168 00000100:00100000:0.0:1767320161.220882:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999168 00000100:00100000:0.0:1767320161.220930:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1547 00000100:00100000:0.0:1767320161.220939:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982999168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320161.220985:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163982999168:12345-0@lo:400:kworker.0 Request processed in 48us (181us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320161.220992:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1547 00010000:00080000:0.0:1767320161.221322:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed6a00 x1853163982999168/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320177 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320161.221335:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163982999168:0@lo:400:kworker.0 00000100:00080000:2.0:1767320161.221581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320161.221595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982998912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320161.221640:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.221688:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982999040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320161.221710:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.221733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982999296:0@lo:38:kworker.0 00000100:00100000:2.0:1767320161.221752:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.221765:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982999424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320161.221786:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.221805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982999552:0@lo:38:kworker.0 00000100:00100000:2.0:1767320161.221824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320161.221850:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982998912 00000100:00100000:2.0:1767320161.221855:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982998912 00000100:00100000:2.0:1767320161.221884:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999040 00000100:00100000:2.0:1767320161.221887:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999040 00000100:00100000:2.0:1767320161.221895:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999296 00000100:00100000:2.0:1767320161.221897:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999296 00000100:00100000:2.0:1767320161.221904:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999552 00000100:00100000:2.0:1767320161.221906:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999552 00000100:00100000:2.0:1767320161.221915:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1767320161.221921:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982998912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320161.221956:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163982998912:12345-0@lo:400:kworker.0 Request processed in 35us (316us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320161.221961:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1767320161.221975:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2044 00000100:00100000:2.0:1767320161.221979:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320161.221998:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320161.222250:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999040:12345-0@lo:38:kworker.0 Request processed in 268us (537us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320161.222257:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2044 00000100:00100000:2.0:1767320161.222294:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2045 00000100:00100000:2.0:1767320161.222299:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999296:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320161.222309:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320161.222334:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999296:12345-0@lo:38:kworker.0 Request processed in 33us (580us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320161.222339:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2045 00000100:00100000:2.0:1767320161.222349:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046 00000100:00100000:2.0:1767320161.222352:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320161.222360:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320161.222393:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163982999552:12345-0@lo:38:kworker.0 Request processed in 27us (556us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320161.222398:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046 00000100:00080000:2.0:1767320161.222415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320161.222436:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed4a80 x1853163982998912/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320177 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320161.222447:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163982998912:0@lo:400:kworker.0 00000100:00080000:2.0:1767320161.222471:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320161.222476:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320161.222479:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982999040:0@lo:38:kworker.0 00000100:00080000:2.0:1767320161.222489:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320161.222492:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320161.222495:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163982999296:0@lo:38:kworker.0 00000100:00080000:2.0:1767320161.222516:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320161.222520:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320161.222523:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163982999552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320161.222890:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999424 00000100:00100000:1.0:1767320161.222898:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999424 00000100:00100000:1.0:1767320161.222939:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 536 00000100:00100000:1.0:1767320161.222945:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982999424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320161.222956:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320161.222997:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163982999424:12345-0@lo:38:kworker.0 Request processed in 51us (1209us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320161.223003:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 536 00000100:00080000:1.0:1767320161.223040:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320161.223045:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320161.223048:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163982999424:0@lo:38:kworker.0 00010000:00010000:1.0:1767320161.412497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320162.192870:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767320162.192902:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915665664 offset 0 length 520: 4 00000100:00100000:2.0:1767320162.192923:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320162.193156:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665920 00000100:00100000:0.0:1767320162.193165:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665920 00000100:00080000:2.0:1767320162.193258:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320162.193349:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915665792 00000100:00100000:0.0:1767320162.193353:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915665792 00000100:00100000:0.0:1767320162.200848:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1549 00000100:00100000:0.0:1767320162.200873:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915665920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320162.200962:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915665920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (8100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320162.200972:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1549 00000100:00100000:0.0:1767320162.201013:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2047 00000100:00100000:0.0:1767320162.201020:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915665792:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320162.201041:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320162.201072:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915665792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 50us (8146us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320162.201081:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2047 00010000:00010000:2.0:1767320162.437226:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320163.461135:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320164.484853:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320165.316607:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320140/real 1767320140] req@ffff93460a581f80 x1853163982995200/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320165 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320165.316643:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a581f80 x1853163982995200/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320165 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320165.316663:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320165.316687:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320165.316691:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982995200:0@lo:250:kworker.0 00000100:00080000:3.0:1767320165.316750:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320165.316756:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320165.316758:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320165.316763:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5389 00000100:00080000:3.0:1767320165.316767:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320165.316773:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320165.316815:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163982999680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320165.316869:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320165.316872:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320165.316874:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320165.316878:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5409 00000100:00080000:3.0:1767320165.316880:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320165.316883:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320165.316897:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163982999936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320165.316925:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320165.316929:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320165.316932:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320165.316935:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5409 00000100:00080000:3.0:1767320165.316938:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320165.316942:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320165.316954:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0a80 x1853163983000192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320165.316962:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320165.316965:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320165.316966:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320165.316969:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5409 00000100:00080000:3.0:1767320165.316972:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320165.316975:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320165.316987:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2680 x1853163983000320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320165.317008:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320165.317011:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320165.317013:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983000064:0@lo:400:kworker.0 00000100:00080000:3.0:1767320165.317013:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320165.317016:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5409 00000100:00080000:3.0:1767320165.317018:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320165.317021:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320165.317030:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163983000704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320165.317073:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320165.317133:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320165.317139:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983000448:0@lo:400:kworker.0 00000100:00100000:1.0:1767320165.317166:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320165.317182:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000064 00000100:00100000:3.0:1767320165.317189:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000064 00000100:00100000:1.0:1767320165.317194:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000448 00000100:00100000:1.0:1767320165.317197:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000448 00000100:00100000:1.0:1767320165.317236:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 961 00000100:00100000:3.0:1767320165.317236:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1550 00000100:00100000:1.0:1767320165.317245:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983000448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320165.317245:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983000064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320165.317318:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983000448:12345-0@lo:400:kworker.0 Request processed in 63us (139us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320165.317319:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983000064:12345-0@lo:400:kworker.0 Request processed in 63us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320165.317327:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 961 00000100:00100000:3.0:1767320165.317327:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1550 00000100:00100000:1.0:1767320165.317369:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983000448:0@lo:400:kworker.0 00010000:00080000:1.0:1767320165.317380:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0700 x1853163983000064/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320181 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320165.317388:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983000064:0@lo:400:kworker.0 00000100:00080000:2.0:1767320165.317408:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320165.317413:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320165.317417:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982999680:0@lo:250:kworker.0 00000400:00000100:1.0:1767320165.317436:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163982999680 offset 0 length 520: 4 00000100:00100000:1.0:1767320165.317455:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982999936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320165.317499:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320165.317525:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983000192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320165.317546:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320165.317561:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983000320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320165.317581:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320165.317606:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983000704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320165.317625:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320165.317727:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999936 00000100:00100000:3.0:1767320165.317732:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999936 00000100:00100000:3.0:1767320165.317762:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000192 00000100:00100000:3.0:1767320165.317766:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000192 00000100:00100000:3.0:1767320165.317774:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000704 00000100:00100000:3.0:1767320165.317777:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000704 00000100:00100000:3.0:1767320165.317786:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2048 00000100:00100000:3.0:1767320165.317791:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982999936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320165.317818:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320165.317852:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163982999936:12345-0@lo:38:kworker.0 Request processed in 60us (353us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320165.317859:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2048 00000100:00100000:3.0:1767320165.317871:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049 00000100:00100000:3.0:1767320165.317875:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983000192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320165.317884:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320165.317906:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983000192:12345-0@lo:38:kworker.0 Request processed in 29us (358us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320165.317912:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049 00000100:00100000:3.0:1767320165.317923:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2050 00000100:00100000:3.0:1767320165.317927:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983000704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320165.317935:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320165.317954:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983000704:12345-0@lo:38:kworker.0 Request processed in 26us (328us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320165.318010:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2050 00000100:00100000:0.0:1767320165.318125:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982999808:0@lo:400:kworker.0 00000100:00100000:2.0:1767320165.318150:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000320 00000100:00100000:2.0:1767320165.318155:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000320 00000100:00100000:0.0:1767320165.318156:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320165.318178:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 537 00000100:00100000:2.0:1767320165.318183:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983000320:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320165.318188:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320165.318193:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983000576:0@lo:400:kworker.0 00010000:00020000:2.0:1767320165.318193:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320165.318213:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320165.318218:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983000320:12345-0@lo:38:kworker.0 Request processed in 35us (636us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320165.318224:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 537 00000100:00100000:2.0:1767320165.318241:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163982999808 00000100:00100000:2.0:1767320165.318244:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163982999808 00000100:00100000:0.0:1767320165.318251:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000576 00000100:00100000:0.0:1767320165.318255:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000576 00000100:00100000:2.0:1767320165.318286:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1551 00000100:00100000:0.0:1767320165.318290:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 962 00000100:00100000:2.0:1767320165.318292:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982999808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320165.318295:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983000576:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320165.318383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320165.318387:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320165.318429:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983000576:12345-0@lo:400:kworker.0 Request processed in 132us (212us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320165.318435:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 962 00000100:00080000:1.0:1767320165.318457:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320165.318463:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320165.318471:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163982999936:0@lo:38:kworker.0 00000100:00080000:1.0:1767320165.318481:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320165.318485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320165.318488:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983000192:0@lo:38:kworker.0 00000100:00080000:1.0:1767320165.318495:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320165.318498:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320165.318501:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983000320:0@lo:38:kworker.0 00000100:00080000:1.0:1767320165.318508:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320165.318511:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320165.318514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983000704:0@lo:38:kworker.0 00000100:00100000:2.0:1767320165.318712:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163982999808:12345-0@lo:400:kworker.0 Request processed in 420us (554us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320165.318720:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1551 00000100:00100000:0.0:1767320165.318853:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983000576:0@lo:400:kworker.0 00010000:00080000:0.0:1767320165.318867:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0000 x1853163982999808/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320181 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320165.318877:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163982999808:0@lo:400:kworker.0 00010000:00010000:2.0:1767320165.508504:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320165.828702:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320165.828727:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983000832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320165.828795:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320165.828865:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000832 00000100:00100000:3.0:1767320165.828873:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000832 00000100:00100000:3.0:1767320165.828921:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1042 00000100:00100000:3.0:1767320165.828930:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983000832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320165.828957:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320165.829005:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983000832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320165.829012:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1042 00000100:00100000:1.0:1767320165.829038:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983000832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320165.829695:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983000960:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320165.829729:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320165.829835:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983000960 00000100:00100000:2.0:1767320165.829840:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983000960 00000100:00100000:2.0:1767320165.829907:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043 00000100:00100000:2.0:1767320165.829915:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983000960:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320165.829942:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320165.829986:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983000960:12345-0@lo:13:osp-pre-0-1.0 Request processed in 72us (256us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320165.829992:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043 00000100:00100000:3.0:1767320165.830051:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983000960:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320166.532633:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320167.308452:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320167.308589:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320167.309121:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666048 00000100:00100000:3.0:1767320167.309134:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666048 00000100:00100000:3.0:1767320167.309186:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666176 00000100:00100000:3.0:1767320167.309190:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666176 00000100:00100000:3.0:1767320167.309347:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1552 00000100:00100000:3.0:1767320167.309357:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915666176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320167.309451:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915666176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (861us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320167.309459:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1552 00000100:00100000:3.0:1767320167.309478:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2051 00000100:00100000:3.0:1767320167.309482:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915666048:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320167.309503:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320167.309529:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915666048:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (1091us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320167.309536:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2051 00000100:00080000:2.0:1767320167.310160:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320167.556657:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320168.581018:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320169.605019:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320170.436981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320170.436992:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767320170.437077:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983001088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320170.437162:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320170.437366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320170.437370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320170.437374:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320170.437379:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5413 00000100:00080000:0.0:1767320170.437383:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320170.437389:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320170.437411:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7480 x1853163983001216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320170.437438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320170.437441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320170.437443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320170.437446:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5413 00000100:00080000:0.0:1767320170.437448:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320170.437451:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320170.437459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5180 x1853163983001472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320170.437467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320170.437470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320170.437472:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320170.437475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5413 00000100:00080000:0.0:1767320170.437476:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320170.437479:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320170.437487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4e00 x1853163983001600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320170.437554:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001088 00000100:00100000:1.0:1767320170.437562:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001088 00000100:00100000:1.0:1767320170.437610:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1553 00000100:00100000:1.0:1767320170.437619:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983001088:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.437790:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983001856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.437823:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320170.437864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320170.437896:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983001344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.437921:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.437939:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320170.437944:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983001728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.437963:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.437988:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001856 00000100:00100000:2.0:1767320170.437992:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001856 00000100:00100000:2.0:1767320170.438022:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001728 00000100:00100000:2.0:1767320170.438025:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001728 00000100:00100000:2.0:1767320170.438044:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963 00000100:00100000:2.0:1767320170.438051:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983001856:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.438085:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983001856:12345-0@lo:400:kworker.0 Request processed in 35us (261us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320170.438092:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963 00000100:00100000:2.0:1767320170.438123:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 964 00000100:00100000:2.0:1767320170.438130:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983001728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.438161:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983001728:12345-0@lo:400:kworker.0 Request processed in 32us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320170.438166:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 964 00000100:00100000:2.0:1767320170.438183:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001344 00000100:00100000:2.0:1767320170.438186:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001344 00000100:00100000:2.0:1767320170.438222:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1554 00000100:00100000:2.0:1767320170.438230:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983001344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.438257:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983001344:12345-0@lo:400:kworker.0 Request processed in 30us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320170.438262:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1554 00000100:00100000:2.0:1767320170.438328:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983001856:0@lo:400:kworker.0 00000100:00100000:2.0:1767320170.438353:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983001728:0@lo:400:kworker.0 00010000:00080000:2.0:1767320170.438364:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed5880 x1853163983001344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320186 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320170.438372:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983001344:0@lo:400:kworker.0 00000100:00080000:2.0:1767320170.438387:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320170.438390:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320170.438392:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320170.438401:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983001216:0@lo:38:kworker.0 00000100:00100000:2.0:1767320170.438424:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.438441:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983001472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320170.438458:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.438472:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983001600:0@lo:38:kworker.0 00000100:00100000:2.0:1767320170.438491:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.438522:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001600 00000100:00100000:2.0:1767320170.438526:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001600 00000100:00100000:2.0:1767320170.438758:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 538 00000100:00100000:2.0:1767320170.438763:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983001600:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320170.438783:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320170.438819:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983001600:12345-0@lo:38:kworker.0 Request processed in 55us (327us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320170.438825:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 538 00000100:00100000:2.0:1767320170.438845:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001216 00000100:00100000:2.0:1767320170.438849:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001216 00000100:00100000:2.0:1767320170.438869:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001472 00000100:00100000:2.0:1767320170.438871:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001472 00000100:00100000:2.0:1767320170.438884:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052 00000100:00100000:2.0:1767320170.438888:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983001216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320170.438897:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320170.438919:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983001216:12345-0@lo:38:kworker.0 Request processed in 31us (495us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320170.438928:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052 00000100:00100000:2.0:1767320170.438937:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053 00000100:00100000:2.0:1767320170.438941:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983001472:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320170.438949:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320170.438964:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983001472:12345-0@lo:38:kworker.0 Request processed in 22us (505us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320170.438969:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053 00000100:00080000:2.0:1767320170.438995:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320170.438999:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320170.439002:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983001216:0@lo:38:kworker.0 00000100:00080000:2.0:1767320170.439010:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320170.439013:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320170.439015:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983001472:0@lo:38:kworker.0 00000100:00080000:2.0:1767320170.439021:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320170.439024:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320170.439027:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983001600:0@lo:38:kworker.0 00000100:00080000:0.0:1767320170.443700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320170.443712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320170.443718:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320170.443729:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5413 00000100:00080000:0.0:1767320170.443735:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320170.443741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320170.443786:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed5f80 x1853163983001984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320170.444744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983001984:0@lo:38:kworker.0 00000100:00100000:2.0:1767320170.444819:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320170.445369:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983001984 00000100:00100000:0.0:1767320170.445382:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983001984 00000100:00100000:0.0:1767320170.445435:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2054 00000100:00100000:0.0:1767320170.445442:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983001984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320170.445465:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320170.445520:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983001984:12345-0@lo:38:kworker.0 Request processed in 76us (699us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320170.445527:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2054 00000100:00080000:0.0:1767320170.445582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320170.445587:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320170.445590:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983001984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320170.446039:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983001088:12345-0@lo:400:kworker.0 Request processed in 8417us (8873us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320170.446050:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1553 00010000:00080000:3.0:1767320170.449004:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7b80 x1853163983001088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320186 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320170.449022:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983001088:0@lo:400:kworker.0 00010000:00010000:1.0:1767320170.628519:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320170.948976:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983002112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320170.949052:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320170.949146:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002112 00000100:00100000:2.0:1767320170.949154:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002112 00000100:00100000:2.0:1767320170.949222:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044 00000100:00100000:2.0:1767320170.949229:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983002112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320170.949255:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320170.949311:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983002112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (258us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320170.949316:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044 00000100:00100000:2.0:1767320170.949353:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983002112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320170.949678:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320170.949690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983002240:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320170.949720:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320170.949834:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002240 00000100:00100000:1.0:1767320170.949840:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002240 00000100:00100000:1.0:1767320170.949876:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1045 00000100:00100000:1.0:1767320170.949884:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983002240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320170.949901:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320170.949939:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983002240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 57us (219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320170.949944:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1045 00000100:00100000:3.0:1767320170.950075:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983002240:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320171.652662:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320172.429006:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320172.429054:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320172.429154:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666432 00000100:00100000:1.0:1767320172.429167:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666432 00000100:00100000:1.0:1767320172.429416:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666304 00000100:00100000:1.0:1767320172.429420:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666304 00000100:00100000:1.0:1767320172.429448:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1555 00000100:00100000:1.0:1767320172.429460:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915666432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320172.429573:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915666432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (580us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320172.429582:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1555 00000100:00100000:1.0:1767320172.429605:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2055 00000100:00100000:1.0:1767320172.429611:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915666304:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:0.0:1767320172.429746:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:1.0:1767320172.430440:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320172.430508:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915666304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 892us (1449us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320172.430517:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2055 00010000:00010000:2.0:1767320172.676636:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320173.700792:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320174.727113:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320175.556484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320175.556496:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320175.556575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320175.556580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320175.556584:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320175.556592:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5418 00000100:00080000:3.0:1767320175.556598:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320175.556605:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320175.556628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2680 x1853163983002496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320175.556684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320175.556688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320175.556691:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320175.556695:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5418 00000100:00080000:3.0:1767320175.556697:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320175.556701:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320175.556713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163983002752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320175.556723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320175.556726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320175.556728:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320175.556731:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5418 00000100:00080000:3.0:1767320175.556734:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320175.556738:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320175.556750:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af0700 x1853163983002880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320175.556776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320175.556779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320175.556781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320175.556784:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5418 00000100:00080000:3.0:1767320175.556787:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320175.556790:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320175.556801:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1f80 x1853163983003264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320175.556850:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983003008:0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.556917:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320175.556977:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320175.556984:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983002624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.557011:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320175.557113:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983003136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320175.557181:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320175.557218:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320175.557224:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983002368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320175.557247:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320175.557425:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983002496:0@lo:38:kworker.0 00000100:00100000:1.0:1767320175.557456:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320175.557485:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983002752:0@lo:38:kworker.0 00000100:00100000:1.0:1767320175.557505:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320175.557522:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983002880:0@lo:38:kworker.0 00000100:00100000:1.0:1767320175.557544:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320175.557582:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983003264:0@lo:38:kworker.0 00000100:00100000:1.0:1767320175.557604:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320175.557636:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002624 00000100:00100000:1.0:1767320175.557643:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002624 00000100:00100000:1.0:1767320175.557687:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002368 00000100:00100000:1.0:1767320175.557691:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002368 00000100:00100000:1.0:1767320175.557705:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002496 00000100:00100000:1.0:1767320175.557708:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002496 00000100:00100000:1.0:1767320175.557718:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002752 00000100:00100000:1.0:1767320175.557720:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002752 00000100:00100000:1.0:1767320175.557747:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003264 00000100:00100000:1.0:1767320175.557751:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003264 00000100:00100000:1.0:1767320175.557764:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1556 00000100:00100000:1.0:1767320175.557773:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983002624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320175.557825:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983002624:12345-0@lo:400:kworker.0 Request processed in 53us (813us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320175.557833:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1556 00000100:00100000:1.0:1767320175.557850:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1557 00000100:00100000:1.0:1767320175.557856:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983002368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320175.557886:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983002368:12345-0@lo:400:kworker.0 Request processed in 30us (637us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320175.557893:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1557 00000100:00100000:1.0:1767320175.557905:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2056 00000100:00100000:1.0:1767320175.557909:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983002496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320175.557928:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320175.557961:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983002496:12345-0@lo:38:kworker.0 Request processed in 49us (503us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320175.557967:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2056 00000100:00100000:1.0:1767320175.557979:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2057 00000100:00100000:1.0:1767320175.557983:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983002752:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320175.557992:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320175.558016:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983002752:12345-0@lo:38:kworker.0 Request processed in 32us (510us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320175.558021:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2057 00000100:00100000:1.0:1767320175.558032:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2058 00000100:00100000:1.0:1767320175.558037:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983003264:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320175.558046:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320175.558068:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983003264:12345-0@lo:38:kworker.0 Request processed in 30us (462us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320175.558073:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2058 00000100:00100000:1.0:1767320175.558091:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983002880 00000100:00100000:1.0:1767320175.558095:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983002880 00000100:00100000:1.0:1767320175.558112:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 539 00000100:00100000:1.0:1767320175.558130:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983002880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320175.558139:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320175.558166:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983002880:12345-0@lo:38:kworker.0 Request processed in 35us (621us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320175.558171:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 539 00000100:00080000:1.0:1767320175.558218:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320175.558223:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320175.558227:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983002496:0@lo:38:kworker.0 00000100:00080000:1.0:1767320175.558235:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320175.558239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320175.558241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983002752:0@lo:38:kworker.0 00000100:00080000:1.0:1767320175.558248:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320175.558251:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320175.558254:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983002880:0@lo:38:kworker.0 00000100:00080000:1.0:1767320175.558261:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320175.558264:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320175.558266:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983003264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320175.558639:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320175.558663:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3480 x1853163983002368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320191 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320175.558676:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983002368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.559013:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003008 00000100:00100000:3.0:1767320175.559018:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003008 00000100:00100000:3.0:1767320175.559072:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003136 00000100:00100000:3.0:1767320175.559075:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003136 00000100:00100000:3.0:1767320175.559094:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965 00000100:00100000:3.0:1767320175.559100:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983003008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.559153:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983003008:12345-0@lo:400:kworker.0 Request processed in 53us (2236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320175.559159:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965 00000100:00100000:3.0:1767320175.559173:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 966 00000100:00100000:3.0:1767320175.559179:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983003136:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.559220:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983003136:12345-0@lo:400:kworker.0 Request processed in 42us (2039us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320175.559227:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 966 00010000:00080000:3.0:1767320175.559255:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af0a80 x1853163983002624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320191 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320175.559294:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983002624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320175.559308:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983003008:0@lo:400:kworker.0 00000100:00080000:0.0:1767320175.559443:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320175.559446:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320175.559466:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983003136:0@lo:400:kworker.0 00010000:00010000:2.0:1767320175.750519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320176.068538:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320176.069925:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983003392:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320176.069994:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320176.071695:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983003520:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320176.071757:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320176.082450:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003392 00000100:00100000:3.0:1767320176.082473:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003392 00000100:00100000:3.0:1767320176.082522:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003520 00000100:00100000:3.0:1767320176.082526:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003520 00000100:00100000:3.0:1767320176.082546:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046 00000100:00100000:3.0:1767320176.082555:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983003392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320176.082587:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320176.082665:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983003392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 112us (12670us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320176.082681:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046 00000100:00100000:3.0:1767320176.082699:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047 00000100:00100000:3.0:1767320176.082705:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983003520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320176.082719:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320176.082754:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983003520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (10997us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320176.082760:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047 00000100:00100000:2.0:1767320176.083314:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983003392:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320176.084707:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983003520:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320176.773141:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320177.554807:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320177.554853:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320177.554872:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666688 00000100:00100000:2.0:1767320177.554882:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666688 00000100:00100000:2.0:1767320177.554960:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666560 00000100:00100000:2.0:1767320177.554965:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666560 00000100:00100000:2.0:1767320177.554984:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1558 00000100:00100000:2.0:1767320177.554994:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915666688:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320177.555067:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915666688:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (271us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320177.555076:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1558 00000100:00100000:2.0:1767320177.555104:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2059 00000100:00100000:2.0:1767320177.555108:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915666560:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320177.555128:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320177.555156:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915666560:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (304us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320177.555162:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2059 00000100:00080000:1.0:1767320177.555232:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320177.798740:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320178.822345:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320179.844645:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320180.676391:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320180.676400:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320180.676459:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320180.676463:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320180.676466:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320180.676475:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5424 00000100:00080000:3.0:1767320180.676480:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320180.676486:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320180.676598:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983003648:0@lo:400:kworker.0 00000100:00100000:0.0:1767320180.676682:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.676791:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003648 00000100:00100000:2.0:1767320180.676813:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003648 00000100:00100000:2.0:1767320180.676885:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1559 00000100:00100000:2.0:1767320180.676895:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983003648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320180.676912:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1500 x1853163983003776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320180.676947:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320180.676951:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983003648:12345-0@lo:400:kworker.0 Request processed in 58us (269us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320180.676952:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320180.676954:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320180.676958:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1559 00000100:00080000:3.0:1767320180.676958:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5424 00000100:00080000:3.0:1767320180.676960:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320180.676964:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320180.676976:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2300 x1853163983004032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320180.676985:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320180.676987:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320180.676989:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320180.676993:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5424 00000100:00080000:3.0:1767320180.676995:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320180.676998:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320180.677009:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2a00 x1853163983004160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320180.677050:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320180.677053:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320180.677055:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320180.677058:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5424 00000100:00080000:3.0:1767320180.677061:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320180.677064:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320180.677075:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163983004544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320180.677100:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983004288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320180.677172:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983003904:0@lo:400:kworker.0 00000100:00080000:1.0:1767320180.677262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320180.677291:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983004416:0@lo:400:kworker.0 00000100:00100000:2.0:1767320180.682160:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.682292:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003904 00000100:00100000:2.0:1767320180.682301:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003904 00000100:00100000:2.0:1767320180.682366:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1560 00000100:00100000:2.0:1767320180.682376:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983003904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320180.682435:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983003904:12345-0@lo:400:kworker.0 Request processed in 61us (283us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320180.682448:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1560 00010000:00080000:2.0:1767320180.682493:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a3b80 x1853163983003904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320196 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320180.682508:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983003904:0@lo:400:kworker.0 00000100:00100000:2.0:1767320180.682549:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983003776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320180.682578:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.682606:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983004032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320180.682627:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.682643:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983004160:0@lo:38:kworker.0 00000100:00100000:2.0:1767320180.682666:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.682686:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983004544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320180.682705:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.682732:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004160 00000100:00100000:2.0:1767320180.682735:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004160 00000100:00100000:2.0:1767320180.682753:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 540 00000100:00100000:2.0:1767320180.682758:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983004160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320180.682775:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:0.0:1767320180.685718:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0700 x1853163983003648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320196 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320180.685739:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983003648:0@lo:400:kworker.0 00000100:00080000:0.0:1767320180.685874:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320180.685915:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983003776 00000100:00100000:0.0:1767320180.685922:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983003776 00000100:00100000:0.0:1767320180.685962:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004032 00000100:00100000:0.0:1767320180.685966:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004032 00000100:00100000:0.0:1767320180.685979:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004544 00000100:00100000:0.0:1767320180.685982:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004544 00000100:00100000:0.0:1767320180.685997:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2060 00000100:00100000:0.0:1767320180.686004:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983003776:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320180.686494:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983004160:12345-0@lo:38:kworker.0 Request processed in 3731us (3823us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320180.686505:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 540 00000100:00100000:2.0:1767320180.686540:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061 00000100:00100000:2.0:1767320180.686544:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983004032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320180.687223:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320180.687227:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320180.687313:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983003776:12345-0@lo:38:kworker.0 Request processed in 1308us (4733us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320180.687323:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2060 00000100:00100000:0.0:1767320180.687349:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2062 00000100:00100000:0.0:1767320180.687355:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983004544:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320180.687365:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320180.687393:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983004544:12345-0@lo:38:kworker.0 Request processed in 38us (4687us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320180.687400:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2062 00000100:00100000:2.0:1767320180.687441:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983004032:12345-0@lo:38:kworker.0 Request processed in 894us (4812us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320180.687447:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061 00000100:00080000:2.0:1767320180.687486:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320180.687491:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320180.687495:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983003776:0@lo:38:kworker.0 00000100:00080000:2.0:1767320180.687507:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320180.687511:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320180.687514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983004032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320180.687525:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320180.687528:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320180.687530:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983004160:0@lo:38:kworker.0 00000100:00080000:2.0:1767320180.687541:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320180.687544:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320180.687546:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983004544:0@lo:38:kworker.0 00000100:00100000:3.0:1767320180.690335:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320180.691353:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320180.691504:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004288 00000100:00100000:1.0:1767320180.691520:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004288 00000100:00100000:1.0:1767320180.691597:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004416 00000100:00100000:1.0:1767320180.691600:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004416 00000100:00100000:1.0:1767320180.691627:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 967 00000100:00100000:1.0:1767320180.691636:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983004288:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320180.691711:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983004288:12345-0@lo:400:kworker.0 Request processed in 77us (1385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320180.691718:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 967 00000100:00100000:1.0:1767320180.691734:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968 00000100:00100000:1.0:1767320180.691739:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983004416:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320180.691767:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983004416:12345-0@lo:400:kworker.0 Request processed in 28us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320180.691772:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968 00000100:00100000:1.0:1767320180.691903:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983004288:0@lo:400:kworker.0 00000100:00100000:1.0:1767320180.691937:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983004416:0@lo:400:kworker.0 00000100:00080000:0.0:1767320180.693580:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320180.693590:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320180.871140:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320181.188897:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320181.188925:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983004672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320181.188996:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320181.189057:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004672 00000100:00100000:0.0:1767320181.189068:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004672 00000100:00100000:0.0:1767320181.189132:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048 00000100:00100000:0.0:1767320181.189143:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983004672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320181.189178:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320181.189244:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983004672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (247us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320181.189252:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048 00000100:00100000:1.0:1767320181.189402:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983004672:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320181.193933:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983004800:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320181.194033:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320181.194231:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004800 00000100:00100000:2.0:1767320181.194241:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004800 00000100:00100000:2.0:1767320181.194327:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049 00000100:00100000:2.0:1767320181.194337:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983004800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320181.194372:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320181.194427:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983004800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (394us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320181.194435:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049 00000100:00100000:0.0:1767320181.194549:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983004800:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320181.905133:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320182.669891:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320182.669965:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320182.670234:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666816 00000100:00100000:1.0:1767320182.670245:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666816 00000100:00100000:1.0:1767320182.670326:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915666944 00000100:00100000:1.0:1767320182.670330:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915666944 00000100:00100000:1.0:1767320182.670383:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1561 00000100:00100000:1.0:1767320182.670393:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915666944:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320182.670470:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915666944:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (503us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320182.670478:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1561 00000100:00100000:1.0:1767320182.670496:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2063 00000100:00100000:1.0:1767320182.670500:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915666816:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320182.670520:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320182.670544:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915666816:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (666us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320182.670551:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2063 00000100:00080000:3.0:1767320182.670952:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320182.916745:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320183.942779:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320184.964534:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320185.796599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320185.796611:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320185.796674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320185.796678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320185.796681:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320185.796689:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5429 00000100:00080000:3.0:1767320185.796695:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320185.796702:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320185.796722:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fed80 x1853163983005056/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320185.796761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320185.796765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320185.796767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320185.796770:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5429 00000100:00080000:3.0:1767320185.796773:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320185.796777:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320185.796789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff480 x1853163983005312/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320185.796797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320185.796800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320185.796802:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320185.796805:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5429 00000100:00080000:3.0:1767320185.796807:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320185.796810:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320185.796820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc700 x1853163983005440/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320185.796840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320185.796844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320185.796845:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320185.796848:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5429 00000100:00080000:3.0:1767320185.796850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320185.796854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320185.796873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc380 x1853163983005824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320185.796898:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983005568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320185.796974:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320185.797048:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320185.797054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983005184:0@lo:400:kworker.0 00000100:00100000:3.0:1767320185.797087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320185.797124:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005184 00000100:00100000:0.0:1767320185.797126:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983005696:0@lo:400:kworker.0 00000100:00100000:3.0:1767320185.797133:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005184 00000100:00100000:0.0:1767320185.797163:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320185.797186:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320185.797193:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983004928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320185.797198:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1562 00000100:00100000:3.0:1767320185.797211:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983005184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320185.797229:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320185.797302:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983005184:12345-0@lo:400:kworker.0 Request processed in 94us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320185.797309:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1562 00000100:00100000:3.0:1767320185.797326:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983004928 00000100:00100000:3.0:1767320185.797330:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983004928 00000100:00100000:3.0:1767320185.797352:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1563 00000100:00100000:3.0:1767320185.797359:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983004928:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320185.797429:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005568 00000100:00100000:2.0:1767320185.797433:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005568 00000100:00080000:0.0:1767320185.797448:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320185.797455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320185.797464:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005696 00000100:00100000:2.0:1767320185.797468:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005696 00000100:00100000:0.0:1767320185.797474:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969 00000100:00100000:0.0:1767320185.797485:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983005568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320185.797497:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 970 00000100:00100000:2.0:1767320185.797506:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983005696:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320185.797687:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983005568:12345-0@lo:400:kworker.0 Request processed in 205us (711us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320185.797695:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969 00000100:00100000:0.0:1767320185.797729:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983005056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320185.797756:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320185.797781:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983005312:0@lo:38:kworker.0 00000100:00100000:0.0:1767320185.797803:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320185.797813:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983005440:0@lo:38:kworker.0 00000100:00100000:0.0:1767320185.797836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320185.797858:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983005824:0@lo:38:kworker.0 00000100:00100000:1.0:1767320185.797917:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005056 00000100:00100000:1.0:1767320185.797923:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005056 00000100:00100000:1.0:1767320185.797955:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005312 00000100:00100000:1.0:1767320185.797959:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005312 00000100:00100000:0.0:1767320185.797964:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320185.797974:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005824 00000100:00100000:1.0:1767320185.797978:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005824 00000100:00100000:1.0:1767320185.797999:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064 00000100:00100000:1.0:1767320185.798005:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983005056:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320185.798009:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065 00000100:00100000:0.0:1767320185.798017:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983005312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320185.798031:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320185.798041:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320185.798072:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983005312:12345-0@lo:38:kworker.0 Request processed in 54us (268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320185.798080:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065 00000100:00100000:1.0:1767320185.798086:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983005056:12345-0@lo:38:kworker.0 Request processed in 78us (327us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320185.798093:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064 00000100:00100000:0.0:1767320185.798101:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2066 00000100:00100000:0.0:1767320185.798107:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983005824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320185.798117:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320185.798141:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320185.798146:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983005824:12345-0@lo:38:kworker.0 Request processed in 37us (183us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320185.798157:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2066 00000100:00080000:1.0:1767320185.798177:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320185.798183:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320185.798188:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983005056:0@lo:38:kworker.0 00000100:00080000:1.0:1767320185.798206:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00080000:0.0:1767320185.798207:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163983005184/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320201 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320185.798213:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320185.798217:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983005312:0@lo:38:kworker.0 00000100:00080000:1.0:1767320185.798226:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320185.798231:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320185.798237:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983005824:0@lo:38:kworker.0 00000100:00100000:0.0:1767320185.798244:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983005184:0@lo:400:kworker.0 00000100:00100000:0.0:1767320185.798263:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983005568:0@lo:400:kworker.0 00000100:00100000:1.0:1767320185.798302:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005440 00000100:00100000:1.0:1767320185.798306:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005440 00000100:00100000:1.0:1767320185.798330:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 541 00000100:00100000:1.0:1767320185.798336:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983005440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320185.798347:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320185.798387:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983005440:12345-0@lo:38:kworker.0 Request processed in 50us (549us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320185.798393:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 541 00000100:00080000:0.0:1767320185.798611:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320185.798615:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320185.798619:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983005440:0@lo:38:kworker.0 00000100:00100000:3.0:1767320185.800054:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983004928:12345-0@lo:400:kworker.0 Request processed in 2689us (2818us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320185.800067:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1563 00000100:00100000:2.0:1767320185.803165:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983005696:12345-0@lo:400:kworker.0 Request processed in 5649us (5990us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320185.803183:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 970 00010000:00080000:0.0:1767320185.810596:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fce00 x1853163983004928/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320201 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320185.810616:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983004928:0@lo:400:kworker.0 00000100:00100000:0.0:1767320185.810648:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983005696:0@lo:400:kworker.0 00010000:00010000:1.0:1767320185.988524:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320186.309132:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983005952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320186.309208:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320186.309368:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983006080:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320186.309394:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320186.309435:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983005952 00000100:00100000:1.0:1767320186.309443:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983005952 00000100:00100000:1.0:1767320186.309475:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006080 00000100:00100000:1.0:1767320186.309477:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006080 00000100:00100000:1.0:1767320186.309486:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050 00000100:00100000:1.0:1767320186.309491:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983005952:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320186.309520:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320186.309563:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983005952:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320186.309616:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050 00000100:00100000:1.0:1767320186.309633:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051 00000100:00100000:1.0:1767320186.309641:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983006080:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320186.309655:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320186.309682:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983006080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 41us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320186.309687:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051 00000100:00100000:1.0:1767320186.309724:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983005952:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320186.309751:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983006080:0@lo:13:osp-pre-1-1.0 00000400:00000100:0.0:1767320186.765576:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915667072 offset 0 length 520: 4 00000100:00100000:2.0:1767320186.770156:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320186.770197:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320186.770442:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667200 00000100:00100000:3.0:1767320186.770453:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667200 00000100:00100000:3.0:1767320186.770501:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667328 00000100:00100000:3.0:1767320186.770504:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667328 00000100:00100000:3.0:1767320186.770553:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1564 00000100:00100000:3.0:1767320186.770563:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915667328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320186.770638:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915667328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320186.770646:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1564 00000100:00100000:3.0:1767320186.770665:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2067 00000100:00100000:3.0:1767320186.770669:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915667200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320186.770689:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320186.770712:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915667200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (568us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320186.770718:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2067 00000100:00080000:2.0:1767320186.771125:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320187.013057:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320188.036784:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320189.060976:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320190.086606:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320190.916433:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.916445:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320190.916514:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.916519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320190.916523:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320190.916531:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5434 00000100:00080000:3.0:1767320190.916536:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320190.916543:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320190.916566:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163983006336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320190.916621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.916625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320190.916627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320190.916631:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5434 00000100:00080000:3.0:1767320190.916633:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320190.916637:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320190.916649:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe300 x1853163983006592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320190.916658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.916661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320190.916663:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320190.916666:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5434 00000100:00080000:3.0:1767320190.916669:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320190.916673:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320190.916685:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163983006720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320190.916706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.916709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320190.916711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320190.916714:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5434 00000100:00080000:3.0:1767320190.916716:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320190.916720:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983006208:0@lo:400:kworker.0 00000100:00080000:3.0:1767320190.916720:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320190.916728:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163983007104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320190.916757:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983006848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320190.916793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320190.916804:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320190.916883:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983006976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.916913:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320190.916943:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006848 00000100:00100000:1.0:1767320190.916950:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006848 00000100:00100000:1.0:1767320190.917016:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006976 00000100:00100000:1.0:1767320190.917020:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006976 00000100:00100000:1.0:1767320190.917044:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971 00000100:00100000:1.0:1767320190.917052:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604770c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983006848:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.917097:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604770c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983006848:12345-0@lo:400:kworker.0 Request processed in 45us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320190.917104:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971 00000100:00100000:1.0:1767320190.917397:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972 00000100:00100000:1.0:1767320190.917406:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983006976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.917434:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983006976:12345-0@lo:400:kworker.0 Request processed in 31us (520us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320190.917441:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972 00000100:00080000:1.0:1767320190.917460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320190.917463:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320190.917497:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006208 00000100:00100000:1.0:1767320190.917501:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006208 00000100:00100000:1.0:1767320190.917531:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1565 00000100:00100000:1.0:1767320190.917536:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983006208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.917567:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983006208:12345-0@lo:400:kworker.0 Request processed in 31us (775us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320190.917573:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1565 00000100:00100000:1.0:1767320190.917592:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983006464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.917616:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320190.917634:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320190.917641:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006464 00000100:00100000:1.0:1767320190.917644:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006464 00000100:00100000:1.0:1767320190.917669:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1566 00000100:00100000:1.0:1767320190.917675:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983006464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.917707:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983006464:12345-0@lo:400:kworker.0 Request processed in 32us (89us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320190.917713:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1566 00000100:00080000:1.0:1767320190.917726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320190.917890:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983006848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320190.918667:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983006976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320190.918760:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320165/real 1767320165] req@ffff934609af3b80 x1853163982999680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320190 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320190.918779:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609af3b80 x1853163982999680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320190 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320190.918793:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983006336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320190.918848:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320190.918888:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983006592:0@lo:38:kworker.0 00000100:00100000:2.0:1767320190.918911:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320190.918932:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983006720:0@lo:38:kworker.0 00000100:00100000:2.0:1767320190.918955:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320190.918990:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983007104:0@lo:38:kworker.0 00000100:00100000:2.0:1767320190.919013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320190.919031:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320190.919047:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320190.919051:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163982999680:0@lo:250:kworker.0 00000100:00100000:2.0:1767320190.919086:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006720 00000100:00100000:2.0:1767320190.919093:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006720 00000100:00100000:2.0:1767320190.919129:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 542 00000100:00100000:2.0:1767320190.919137:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983006720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320190.919159:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320190.919198:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983006720:12345-0@lo:38:kworker.0 Request processed in 60us (240us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320190.919217:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 542 00000100:00100000:2.0:1767320190.919237:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006336 00000100:00100000:2.0:1767320190.919241:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006336 00000100:00100000:2.0:1767320190.919255:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983006592 00000100:00100000:2.0:1767320190.919257:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983006592 00000100:00100000:2.0:1767320190.919314:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007104 00000100:00100000:2.0:1767320190.919317:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007104 00000100:00100000:2.0:1767320190.919329:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2068 00000100:00100000:2.0:1767320190.919334:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983006336:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320190.919343:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320190.919368:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983006336:12345-0@lo:38:kworker.0 Request processed in 34us (520us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320190.919374:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2068 00000100:00100000:2.0:1767320190.919385:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2069 00000100:00100000:2.0:1767320190.919389:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983006592:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320190.919840:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070 00000100:00100000:1.0:1767320190.919848:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983007104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320190.919858:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320190.919891:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983007104:12345-0@lo:38:kworker.0 Request processed in 42us (876us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320190.919896:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070 00000100:00080000:1.0:1767320190.919960:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320190.919965:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320190.919969:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983006336:0@lo:38:kworker.0 00000100:00080000:1.0:1767320190.919992:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320190.919995:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320190.919998:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983006720:0@lo:38:kworker.0 00000100:00080000:1.0:1767320190.920004:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320190.920007:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320190.920010:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983007104:0@lo:38:kworker.0 00010000:00080000:3.0:1767320190.920109:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fea00 x1853163983006464/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320206 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320190.920121:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983006464:0@lo:400:kworker.0 00000100:00080000:3.0:1767320190.920145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320190.920150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320190.920153:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320190.920158:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5413 00000100:00080000:3.0:1767320190.920161:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320190.920166:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320190.920189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fea00 x1853163983007232/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00020000:2.0:1767320190.920553:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320190.920590:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983006592:12345-0@lo:38:kworker.0 Request processed in 1198us (1676us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320190.920597:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2069 00000100:00100000:1.0:1767320190.920759:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983007232:0@lo:250:kworker.0 00000400:00000100:1.0:1767320190.920786:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983007232 offset 0 length 520: 4 00000100:00080000:1.0:1767320190.920816:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320190.920821:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320190.920824:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983006592:0@lo:38:kworker.0 00010000:00080000:0.0:1767320190.922620:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fca80 x1853163983006208/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320206 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320190.922639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983006208:0@lo:400:kworker.0 00010000:00010000:2.0:1767320191.108686:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320191.432026:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320191.432052:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983007360:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320191.432127:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320191.432193:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983007488:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320191.432226:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320191.432328:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007360 00000100:00100000:1.0:1767320191.432338:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007360 00000100:00100000:1.0:1767320191.432389:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007488 00000100:00100000:1.0:1767320191.432393:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007488 00000100:00100000:1.0:1767320191.432452:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052 00000100:00100000:1.0:1767320191.432462:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983007360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320191.432501:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320191.432551:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983007360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (423us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320191.432563:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052 00000100:00100000:1.0:1767320191.432583:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053 00000100:00100000:1.0:1767320191.432591:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983007488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320191.432607:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320191.432635:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983007488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (408us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320191.432641:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053 00000100:00100000:1.0:1767320191.432685:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983007360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320191.432771:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983007488:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320191.888051:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320191.888111:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320191.888215:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667584 00000100:00100000:3.0:1767320191.888229:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667584 00000100:00100000:3.0:1767320191.888391:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667456 00000100:00100000:3.0:1767320191.888395:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667456 00000100:00100000:3.0:1767320191.888421:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1567 00000100:00100000:3.0:1767320191.888431:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915667584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320191.888520:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915667584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (488us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320191.888529:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1567 00000100:00100000:3.0:1767320191.888550:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2071 00000100:00100000:3.0:1767320191.888555:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915667456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320191.888577:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320191.888617:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915667456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (506us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320191.888624:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2071 00000100:00080000:1.0:1767320191.888767:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320192.132860:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320193.169161:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320194.180501:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320195.206193:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320196.040326:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320196.040335:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320196.040394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320196.040398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320196.040402:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320196.040410:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5439 00000100:00080000:1.0:1767320196.040414:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320196.040420:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320196.040453:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163983007744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320196.040592:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983007616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.040654:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.040700:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007616 00000100:00100000:3.0:1767320196.040707:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007616 00000100:00100000:3.0:1767320196.040767:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1568 00000100:00100000:3.0:1767320196.040776:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983007616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.040818:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983007616:12345-0@lo:400:kworker.0 Request processed in 44us (164us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320196.040825:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1568 00000100:00080000:3.0:1767320196.040843:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320196.040868:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3480 x1853163983007616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320212 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320196.040886:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983007616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.040918:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983007744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320196.040939:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.040962:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007744 00000100:00100000:3.0:1767320196.040966:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007744 00000100:00100000:3.0:1767320196.040981:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2072 00000100:00100000:3.0:1767320196.040985:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983007744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320196.041003:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320196.041030:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983007744:12345-0@lo:38:kworker.0 Request processed in 43us (90us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320196.041036:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2072 00000100:00080000:3.0:1767320196.041054:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320196.041058:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320196.041061:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983007744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320196.041082:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983007872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.041101:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.041115:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983007872 00000100:00100000:3.0:1767320196.041118:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983007872 00000100:00100000:3.0:1767320196.041137:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1569 00000100:00100000:3.0:1767320196.041141:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983007872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.041163:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983007872:12345-0@lo:400:kworker.0 Request processed in 21us (60us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320196.041168:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1569 00000100:00080000:3.0:1767320196.041178:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320196.041192:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2680 x1853163983007872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320212 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320196.041202:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983007872:0@lo:400:kworker.0 00000100:00080000:1.0:1767320196.041677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320196.041682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320196.041685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320196.041689:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5439 00000100:00080000:1.0:1767320196.041692:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320196.041696:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320196.041713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2680 x1853163983008000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320196.041727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320196.041730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320196.041732:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320196.041763:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5439 00000100:00080000:1.0:1767320196.041766:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320196.041769:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320196.041781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163983008128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320196.041804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320196.041807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320196.041809:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320196.041812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5439 00000100:00080000:1.0:1767320196.041814:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320196.041817:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320196.041825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580000 x1853163983008512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320196.041898:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983008384:0@lo:400:kworker.0 00000100:00100000:2.0:1767320196.041935:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.042012:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008384 00000100:00100000:1.0:1767320196.042017:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008384 00000100:00100000:1.0:1767320196.042058:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 973 00000100:00100000:1.0:1767320196.042065:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983008384:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.042101:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983008384:12345-0@lo:400:kworker.0 Request processed in 37us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.042107:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 973 00000100:00100000:1.0:1767320196.042136:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983008384:0@lo:400:kworker.0 00000100:00080000:2.0:1767320196.042185:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320196.042620:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983008256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320196.042649:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.042685:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983008000:0@lo:38:kworker.0 00000100:00100000:3.0:1767320196.042706:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.042728:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983008128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320196.042758:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.042782:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983008512:0@lo:38:kworker.0 00000100:00100000:3.0:1767320196.042802:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.042834:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008000 00000100:00100000:3.0:1767320196.042839:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008000 00000100:00100000:3.0:1767320196.042859:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008512 00000100:00100000:3.0:1767320196.042862:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008512 00000100:00100000:3.0:1767320196.042875:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073 00000100:00100000:3.0:1767320196.042879:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983008000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320196.042897:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320196.042929:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983008000:12345-0@lo:38:kworker.0 Request processed in 48us (221us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320196.042935:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073 00000100:00100000:3.0:1767320196.042947:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2074 00000100:00100000:3.0:1767320196.042951:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983008512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320196.042960:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320196.042982:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983008512:12345-0@lo:38:kworker.0 Request processed in 29us (178us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320196.042987:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2074 00000100:00080000:3.0:1767320196.043017:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320196.043021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320196.043024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983008000:0@lo:38:kworker.0 00000100:00080000:3.0:1767320196.043031:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320196.043034:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320196.043037:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983008512:0@lo:38:kworker.0 00000100:00100000:2.0:1767320196.043112:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008128 00000100:00100000:2.0:1767320196.043116:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008128 00000100:00100000:2.0:1767320196.043141:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 543 00000100:00100000:2.0:1767320196.043146:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983008128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320196.043155:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320196.043187:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983008128:12345-0@lo:38:kworker.0 Request processed in 40us (428us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320196.043193:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 543 00000100:00080000:2.0:1767320196.043214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320196.043218:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320196.043221:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983008128:0@lo:38:kworker.0 00000100:00100000:1.0:1767320196.044421:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008256 00000100:00100000:1.0:1767320196.044447:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008256 00000100:00080000:2.0:1767320196.044675:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320196.044760:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 974 00000100:00100000:1.0:1767320196.044769:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983008256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320196.044873:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983008256:12345-0@lo:400:kworker.0 Request processed in 105us (2221us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.044881:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 974 00000100:00100000:3.0:1767320196.045108:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983008256:0@lo:400:kworker.0 00010000:00010000:2.0:1767320196.228438:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320196.549041:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983008640:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320196.549116:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320196.549158:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983008768:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320196.549180:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320196.549299:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008640 00000100:00100000:1.0:1767320196.549307:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008640 00000100:00100000:1.0:1767320196.549359:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008768 00000100:00100000:1.0:1767320196.549363:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008768 00000100:00100000:1.0:1767320196.549379:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1054 00000100:00100000:1.0:1767320196.549387:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983008640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320196.549422:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320196.549491:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983008640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.549499:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1054 00000100:00100000:1.0:1767320196.549517:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055 00000100:00100000:1.0:1767320196.549524:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983008768:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320196.549539:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320196.549565:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983008768:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320196.549570:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055 00000100:00100000:0.0:1767320196.549785:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983008640:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320196.549803:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983008768:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320197.006660:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320197.006770:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667840 00000100:00100000:0.0:1767320197.006780:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667840 00000100:00100000:0.0:1767320197.006849:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1570 00000100:00100000:0.0:1767320197.006866:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915667840:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320197.006949:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915667840:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320197.006958:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1570 00000100:00080000:2.0:1767320197.007189:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320197.010228:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320197.010302:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667712 00000100:00100000:0.0:1767320197.010312:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667712 00000100:00100000:0.0:1767320197.010364:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2075 00000100:00100000:0.0:1767320197.010372:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915667712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320197.010400:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320197.010471:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915667712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 97us (251us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320197.010478:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2075 00010000:00010000:2.0:1767320197.253633:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320198.276885:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320199.300646:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320200.324854:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320201.158462:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320201.158472:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320201.158553:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320201.158557:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320201.158561:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320201.158568:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5444 00000100:00080000:1.0:1767320201.158573:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320201.158580:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320201.158603:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1f80 x1853163983009024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320201.158634:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320201.158637:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320201.158639:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320201.158652:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5444 00000100:00080000:1.0:1767320201.158655:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320201.158660:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320201.158673:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3b80 x1853163983009280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320201.158681:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983009024:0@lo:38:kworker.0 00000100:00080000:1.0:1767320201.158697:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320201.158700:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320201.158702:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320201.158705:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5444 00000100:00080000:1.0:1767320201.158708:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320201.158711:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320201.158725:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965500 x1853163983009408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320201.158752:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320201.158774:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320201.158778:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320201.158780:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320201.158784:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5444 00000100:00080000:1.0:1767320201.158786:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320201.158791:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320201.158796:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983009280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320201.158804:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964e00 x1853163983009792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320201.158824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.158845:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983009408:0@lo:38:kworker.0 00000100:00100000:2.0:1767320201.158869:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320201.158902:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983009536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.158920:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983009792:0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.158942:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.158954:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320201.158979:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320201.158988:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983009664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159013:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009024 00000100:00100000:0.0:1767320201.159015:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.159021:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009024 00000100:00100000:0.0:1767320201.159049:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009536 00000100:00100000:2.0:1767320201.159053:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009280 00000100:00100000:0.0:1767320201.159054:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009536 00000100:00100000:2.0:1767320201.159058:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009280 00000100:00100000:2.0:1767320201.159074:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009792 00000100:00100000:2.0:1767320201.159076:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009792 00000100:00100000:2.0:1767320201.159094:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076 00000100:00100000:2.0:1767320201.159101:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009024:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.159107:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009664 00000100:00100000:0.0:1767320201.159112:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009664 00010000:00020000:2.0:1767320201.159121:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320201.159136:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975 00000100:00100000:0.0:1767320201.159144:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983009536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159166:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009024:12345-0@lo:38:kworker.0 Request processed in 63us (414us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320201.159173:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076 00000100:00100000:2.0:1767320201.159190:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2077 00000100:00100000:2.0:1767320201.159195:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009280:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320201.159201:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983009536:12345-0@lo:400:kworker.0 Request processed in 55us (256us total) trans 0 rc 0/0 00010000:00020000:2.0:1767320201.159208:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320201.159212:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975 00000100:00100000:0.0:1767320201.159235:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 976 00000100:00100000:2.0:1767320201.159241:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009280:12345-0@lo:38:kworker.0 Request processed in 42us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320201.159246:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983009664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159253:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2077 00000100:00100000:2.0:1767320201.159291:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2078 00000100:00100000:2.0:1767320201.159298:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320201.159305:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320201.159318:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983009664:12345-0@lo:400:kworker.0 Request processed in 74us (301us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320201.159325:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 976 00000100:00100000:2.0:1767320201.159331:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983009792:12345-0@lo:38:kworker.0 Request processed in 32us (377us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320201.159337:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2078 00000100:00080000:0.0:1767320201.159352:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320201.159356:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983009152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159372:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320201.159373:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320201.159388:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320201.159394:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983008896:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159406:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320201.159412:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983009664:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159421:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009152 00000100:00100000:2.0:1767320201.159425:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009152 00000100:00100000:0.0:1767320201.159433:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983009536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159446:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983008896 00000100:00100000:2.0:1767320201.159449:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983008896 00000100:00100000:2.0:1767320201.159460:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1571 00000100:00080000:0.0:1767320201.159461:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320201.159465:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983009152:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159495:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983009152:12345-0@lo:400:kworker.0 Request processed in 30us (122us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320201.159499:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1571 00000100:00100000:2.0:1767320201.159509:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1572 00000100:00100000:2.0:1767320201.159516:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983008896:12345-0@lo:400:kworker.0 00010000:00080000:0.0:1767320201.159526:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2680 x1853163983009152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320201.159537:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983009152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320201.159551:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983008896:12345-0@lo:400:kworker.0 Request processed in 34us (141us total) trans 0 rc 0/0 00010000:00080000:0.0:1767320201.159559:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af3480 x1853163983008896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320201.159560:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1572 00000100:00100000:0.0:1767320201.159572:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983008896:0@lo:400:kworker.0 00000100:00080000:2.0:1767320201.159594:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.159600:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.159604:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983009792:0@lo:38:kworker.0 00000100:00080000:2.0:1767320201.159614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.159616:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.159618:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983009280:0@lo:38:kworker.0 00000100:00080000:2.0:1767320201.159626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.159628:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.159631:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983009024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320201.159897:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009408 00000100:00100000:1.0:1767320201.159903:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009408 00000100:00100000:1.0:1767320201.159932:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 544 00000100:00100000:1.0:1767320201.159938:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983009408:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320201.159949:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320201.159994:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983009408:12345-0@lo:38:kworker.0 Request processed in 55us (1124us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320201.160002:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 544 00000100:00080000:2.0:1767320201.160020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320201.160027:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320201.160031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983009408:0@lo:38:kworker.0 00010000:00010000:1.0:1767320201.349651:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320201.669864:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983010048:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320201.669951:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320201.670102:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010048 00000100:00100000:2.0:1767320201.670112:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010048 00000100:00100000:3.0:1767320201.670176:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983009920:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320201.670207:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320201.670704:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983009920 00000100:00100000:1.0:1767320201.670710:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983009920 00000100:00100000:2.0:1767320201.672978:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056 00000100:00100000:2.0:1767320201.673000:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983010048:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320201.673041:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320201.673141:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983010048:12345-0@lo:13:osp-pre-0-1.0 Request processed in 152us (3189us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320201.673149:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056 00000100:00100000:2.0:1767320201.673168:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057 00000100:00100000:2.0:1767320201.673175:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983009920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320201.673191:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320201.673215:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983009920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 41us (3008us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320201.673221:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057 00000100:00100000:0.0:1767320201.673329:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983010048:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320201.673348:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983009920:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320202.128043:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320202.128712:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668096 00000100:00100000:3.0:1767320202.128731:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668096 00000100:00100000:3.0:1767320202.128817:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1573 00000100:00100000:3.0:1767320202.128827:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915668096:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320202.128908:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915668096:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (877us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320202.128921:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1573 00000100:00080000:0.0:1767320202.129148:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320202.131771:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320202.131988:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915667968 00000100:00100000:3.0:1767320202.131999:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915667968 00000100:00100000:3.0:1767320202.132051:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2079 00000100:00100000:3.0:1767320202.132060:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915667968:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320202.132082:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320202.132149:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915667968:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 89us (389us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320202.132156:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2079 00010000:00010000:2.0:1767320202.374413:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320203.398888:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320204.420506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320205.446500:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320206.276406:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.276418:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320206.276498:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.276503:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.276508:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.276519:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.276526:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.276534:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320206.276592:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964700 x1853163983010304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320206.276638:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.276641:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.276644:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320206.276646:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767320206.276664:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.276667:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320206.276673:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983010176:0@lo:400:kworker.0 00000100:00080000:1.0:1767320206.276674:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320206.276694:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51180 x1853163983010560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320206.276705:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.276711:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.276713:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.276717:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.276720:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.276724:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320206.276738:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53800 x1853163983010688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320206.276762:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320206.276764:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320206.276768:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320206.276770:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320206.276774:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5449 00000100:00080000:1.0:1767320206.276790:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320206.276795:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320206.276809:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50700 x1853163983011072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320206.276841:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983010944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320206.276856:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983010816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320206.276882:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320206.276911:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.276930:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010944 00000100:00100000:3.0:1767320206.276938:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010944 00000100:00100000:1.0:1767320206.276942:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320206.276948:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983010432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320206.276981:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320206.277001:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010816 00000100:00100000:3.0:1767320206.277007:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010816 00000100:00100000:1.0:1767320206.277022:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977 00000100:00100000:0.0:1767320206.277088:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010176 00000100:00100000:0.0:1767320206.277093:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010176 00000100:00100000:0.0:1767320206.277124:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010432 00000100:00100000:0.0:1767320206.277128:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010432 00000100:00100000:0.0:1767320206.277149:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1574 00000100:00100000:0.0:1767320206.277156:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983010176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.277214:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983010176:12345-0@lo:400:kworker.0 Request processed in 59us (454us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.277222:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1574 00000100:00100000:0.0:1767320206.277238:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1575 00000100:00100000:0.0:1767320206.277244:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983010432:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320206.277307:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983010432:12345-0@lo:400:kworker.0 Request processed in 63us (327us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.277313:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1575 00000100:00080000:0.0:1767320206.277332:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320206.277337:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320206.277338:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:0.0:1767320206.277405:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964e00 x1853163983010176/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320206.277417:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983010176:0@lo:400:kworker.0 00010000:00080000:0.0:1767320206.277463:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f967800 x1853163983010432/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320206.277473:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983010432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320206.277851:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983010944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320206.277901:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983010944:12345-0@lo:400:kworker.0 Request processed in 870us (1030us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320206.277958:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983010944:0@lo:400:kworker.0 00000100:00100000:2.0:1767320206.278147:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983010304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320206.278185:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320206.278212:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983010560:0@lo:38:kworker.0 00000100:00100000:2.0:1767320206.278233:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320206.278251:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983010688:0@lo:38:kworker.0 00000100:00100000:2.0:1767320206.278289:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320206.278313:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983011072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320206.278334:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320206.278365:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010688 00000100:00100000:2.0:1767320206.278370:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010688 00000100:00100000:2.0:1767320206.278399:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 545 00000100:00100000:2.0:1767320206.278404:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983010688:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320206.278423:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320206.278454:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983010688:12345-0@lo:38:kworker.0 Request processed in 49us (164us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320206.278460:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 545 00000100:00100000:2.0:1767320206.278488:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010304 00000100:00100000:2.0:1767320206.278502:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010304 00000100:00100000:2.0:1767320206.278522:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983010560 00000100:00100000:2.0:1767320206.278525:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983010560 00000100:00100000:2.0:1767320206.278538:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011072 00000100:00100000:2.0:1767320206.278540:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011072 00000100:00100000:2.0:1767320206.278561:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2080 00000100:00100000:2.0:1767320206.278566:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983010304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320206.278576:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320206.278602:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983010304:12345-0@lo:38:kworker.0 Request processed in 34us (416us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320206.278608:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2080 00000100:00100000:2.0:1767320206.278619:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2081 00000100:00100000:2.0:1767320206.278623:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983010560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320206.278632:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320206.278673:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983010560:12345-0@lo:38:kworker.0 Request processed in 47us (439us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320206.278678:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2081 00000100:00100000:2.0:1767320206.278690:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082 00000100:00100000:2.0:1767320206.278694:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983011072:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320206.278704:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320206.278726:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983011072:12345-0@lo:38:kworker.0 Request processed in 30us (391us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320206.278731:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082 00000100:00080000:2.0:1767320206.278763:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320206.278768:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320206.278772:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983010304:0@lo:38:kworker.0 00000100:00080000:2.0:1767320206.278781:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320206.281482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320206.281569:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983010560:0@lo:38:kworker.0 00000100:00080000:2.0:1767320206.281618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320206.281623:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320206.281626:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983010688:0@lo:38:kworker.0 00000100:00080000:2.0:1767320206.281640:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320206.281644:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320206.281674:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983011072:0@lo:38:kworker.0 00000100:00100000:3.0:1767320206.285149:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 978 00000100:00100000:3.0:1767320206.285183:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983010816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320206.285330:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983010816:12345-0@lo:400:kworker.0 Request processed in 166us (8419us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320206.285339:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 978 00000100:00100000:2.0:1767320206.285834:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983010816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320206.286128:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977 00010000:00010000:2.0:1767320206.468398:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320206.790268:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320206.790319:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983011200:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320206.790414:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320206.790480:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011200 00000100:00100000:1.0:1767320206.790490:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011200 00000100:00100000:1.0:1767320206.790556:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058 00000100:00100000:1.0:1767320206.790566:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983011200:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320206.790600:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320206.790601:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983011328:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320206.790632:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320206.790644:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983011200:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320206.790651:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058 00000100:00100000:1.0:1767320206.790671:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011328 00000100:00100000:1.0:1767320206.790674:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011328 00000100:00100000:1.0:1767320206.790719:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059 00000100:00100000:1.0:1767320206.790725:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983011328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320206.790742:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320206.790762:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983011200:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320206.790767:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983011328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (135us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320206.790781:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059 00000100:00100000:2.0:1767320206.790975:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983011328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320207.244086:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320207.244621:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668352 00000100:00100000:0.0:1767320207.244632:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668352 00000100:00100000:0.0:1767320207.244730:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1576 00000100:00100000:0.0:1767320207.244743:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915668352:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320207.244855:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915668352:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (784us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320207.244862:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1576 00000100:00080000:2.0:1767320207.245040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320207.250387:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320207.250489:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668224 00000100:00100000:3.0:1767320207.250501:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668224 00000100:00100000:3.0:1767320207.250564:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2083 00000100:00100000:3.0:1767320207.250573:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915668224:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320207.250600:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320207.250679:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915668224:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 105us (303us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320207.250686:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2083 00010000:00010000:1.0:1767320207.492478:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320208.516503:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320209.540390:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320210.564505:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320211.400211:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320211.400222:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320211.400331:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320211.400336:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320211.400340:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320211.400349:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5454 00000100:00080000:3.0:1767320211.400354:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320211.400362:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320211.400385:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163983011584/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320211.400436:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320211.400440:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320211.400442:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320211.400445:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5454 00000100:00080000:3.0:1767320211.400448:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320211.400462:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320211.400474:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983011840/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320211.400483:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320211.400487:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320211.400488:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320211.400492:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5454 00000100:00080000:3.0:1767320211.400494:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320211.400497:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320211.400508:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163983011968/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320211.400529:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320211.400533:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320211.400534:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320211.400538:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5454 00000100:00080000:3.0:1767320211.400540:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320211.400544:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320211.400553:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911880 x1853163983012352/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320211.400588:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983011712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.401042:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983011584:0@lo:38:kworker.0 00000100:00100000:0.0:1767320211.401135:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983012096:0@lo:400:kworker.0 00000100:00100000:2.0:1767320211.401814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983012224:0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.404304:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.404413:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983011840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320211.404485:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.404518:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983011968:0@lo:38:kworker.0 00000100:00100000:1.0:1767320211.404547:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.404579:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983012352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320211.404620:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.405084:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011584 00000100:00100000:1.0:1767320211.405094:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011584 00000100:00100000:1.0:1767320211.405133:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011840 00000100:00100000:1.0:1767320211.405137:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011840 00000100:00100000:1.0:1767320211.405151:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012352 00000100:00100000:1.0:1767320211.405154:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012352 00000100:00100000:1.0:1767320211.405168:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2084 00000100:00100000:1.0:1767320211.405177:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983011584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320211.405205:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320211.405305:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983011584:12345-0@lo:38:kworker.0 Request processed in 126us (1011us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.405312:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2084 00000100:00100000:1.0:1767320211.405332:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085 00000100:00100000:1.0:1767320211.405336:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983011840:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320211.405345:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320211.405369:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983011840:12345-0@lo:38:kworker.0 Request processed in 32us (895us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.405375:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085 00000100:00100000:1.0:1767320211.405385:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2086 00000100:00100000:1.0:1767320211.405388:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983012352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320211.405396:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320211.405416:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983012352:12345-0@lo:38:kworker.0 Request processed in 25us (795us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.405421:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2086 00000100:00080000:1.0:1767320211.405477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320211.405485:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320211.405489:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983011584:0@lo:38:kworker.0 00000100:00080000:1.0:1767320211.405499:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320211.405502:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320211.405504:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983011840:0@lo:38:kworker.0 00000100:00080000:1.0:1767320211.405512:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320211.405515:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320211.405517:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983012352:0@lo:38:kworker.0 00000100:00100000:1.0:1767320211.405572:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983011456:0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.406124:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320211.406168:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011456 00000100:00100000:1.0:1767320211.406173:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011456 00000100:00100000:1.0:1767320211.406240:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1577 00000100:00100000:1.0:1767320211.406249:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983011456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.406321:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983011456:12345-0@lo:400:kworker.0 Request processed in 74us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320211.406327:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1577 00010000:00080000:1.0:1767320211.406362:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910a80 x1853163983011456/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320227 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320211.406376:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983011456:0@lo:400:kworker.0 00000100:00080000:1.0:1767320211.406426:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320211.406441:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011968 00000100:00100000:1.0:1767320211.406445:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011968 00000100:00100000:1.0:1767320211.406479:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 546 00000100:00100000:1.0:1767320211.406484:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983011968:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320211.406498:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320211.406532:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983011968:12345-0@lo:38:kworker.0 Request processed in 48us (1984us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320211.406538:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 546 00000100:00080000:1.0:1767320211.406562:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320211.406566:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320211.406569:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983011968:0@lo:38:kworker.0 00000100:00100000:3.0:1767320211.407443:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320211.407566:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320211.407728:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012096 00000100:00100000:3.0:1767320211.407737:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012096 00000100:00100000:3.0:1767320211.407810:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 979 00000100:00100000:3.0:1767320211.407820:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983012096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320211.407873:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983012096:12345-0@lo:400:kworker.0 Request processed in 55us (306us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320211.407880:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 979 00000100:00080000:0.0:1767320211.408215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320211.408267:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983012096:0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.408486:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983011712 00000100:00100000:1.0:1767320211.408492:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983011712 00000100:00100000:1.0:1767320211.408550:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1578 00000100:00100000:1.0:1767320211.408557:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983011712:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320211.408620:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983011712:12345-0@lo:400:kworker.0 Request processed in 64us (1181us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320211.408626:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1578 00010000:00080000:3.0:1767320211.408981:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912a00 x1853163983011712/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320227 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320211.409005:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983011712:0@lo:400:kworker.0 00000100:00080000:0.0:1767320211.409077:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320211.410546:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320211.410970:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012224 00000100:00100000:3.0:1767320211.410979:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012224 00000100:00100000:3.0:1767320211.411050:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980 00000100:00100000:3.0:1767320211.411060:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983012224:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320211.411136:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983012224:12345-0@lo:400:kworker.0 Request processed in 78us (595us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320211.411143:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980 00000100:00080000:0.0:1767320211.411210:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320211.411576:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983012224:0@lo:400:kworker.0 00010000:00010000:2.0:1767320211.589899:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320211.908693:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983012480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320211.908766:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.908846:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012480 00000100:00100000:2.0:1767320211.908853:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012480 00000100:00100000:2.0:1767320211.908905:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060 00000100:00100000:2.0:1767320211.908914:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983012480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320211.908943:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320211.908992:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983012480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320211.909001:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060 00000100:00100000:0.0:1767320211.909013:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320211.909024:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983012608:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320211.909055:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320211.909122:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012608 00000100:00100000:2.0:1767320211.909126:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012608 00000100:00100000:2.0:1767320211.909151:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061 00000100:00100000:2.0:1767320211.909157:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983012608:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320211.909163:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983012480:0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320211.909176:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320211.909207:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983012608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 50us (151us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320211.909213:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061 00000100:00100000:1.0:1767320211.909522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983012608:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320212.366377:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320212.366610:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668736 00000100:00100000:3.0:1767320212.366620:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668736 00000100:00100000:3.0:1767320212.366701:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1579 00000100:00100000:3.0:1767320212.366711:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915668736:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320212.366780:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915668736:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320212.366787:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1579 00000400:00000100:1.0:1767320212.367096:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915668480 offset 0 length 520: 4 00000100:00100000:1.0:1767320212.367193:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767320212.367349:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320212.367411:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668608 00000100:00100000:2.0:1767320212.367417:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668608 00000100:00100000:2.0:1767320212.367460:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2087 00000100:00100000:2.0:1767320212.367466:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915668608:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320212.367487:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320212.367526:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915668608:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320212.367532:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2087 00010000:00010000:2.0:1767320212.612570:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320213.636606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320214.660511:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320215.492416:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320190/real 1767320190] req@ffff9346100fea00 x1853163983007232/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320215 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320215.492456:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100fea00 x1853163983007232/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320215 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320215.492479:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320215.492516:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320215.492520:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983007232:0@lo:250:kworker.0 00000100:00080000:3.0:1767320215.492610:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320215.492616:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320215.492619:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320215.492624:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5439 00000100:00080000:3.0:1767320215.492628:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320215.492634:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320215.492671:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912a00 x1853163983012736/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320215.492724:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983012736:0@lo:250:kworker.0 00000100:00080000:3.0:1767320215.492731:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320215.492735:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320215.492737:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320215.492741:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320215.492744:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320215.492757:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000400:00000100:1.0:1767320215.492770:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983012736 offset 0 length 520: 4 00000100:00100000:3.0:1767320215.492776:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912d80 x1853163983012992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320215.492810:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320215.492814:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320215.492817:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983012992:0@lo:38:kworker.0 00000100:00080000:3.0:1767320215.492818:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320215.492826:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320215.492830:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320215.492834:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320215.492848:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913b80 x1853163983013248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320215.492859:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320215.492865:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320215.492868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320215.492869:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320215.492877:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320215.492880:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320215.492883:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320215.492897:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913480 x1853163983013376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320215.492908:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983013248:0@lo:38:kworker.0 00000100:00080000:3.0:1767320215.492923:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320215.492928:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320215.492930:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320215.492934:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5459 00000100:00080000:3.0:1767320215.492936:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320215.492941:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320215.492947:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320215.492966:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912300 x1853163983013760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320215.492975:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983013376:0@lo:38:kworker.0 00000100:00100000:3.0:1767320215.492999:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983013120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320215.493023:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320215.493036:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320215.493052:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983012864:0@lo:400:kworker.0 00000100:00100000:1.0:1767320215.493061:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983013760:0@lo:38:kworker.0 00000100:00100000:3.0:1767320215.493064:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320215.493073:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983013504:0@lo:400:kworker.0 00000100:00100000:0.0:1767320215.493094:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320215.493112:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320215.493123:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013376 00000100:00100000:1.0:1767320215.493132:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013376 00000100:00100000:2.0:1767320215.493166:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983013632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320215.493174:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 547 00000100:00100000:1.0:1767320215.493182:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983013376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320215.493203:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320215.497248:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012992 00000100:00100000:2.0:1767320215.497267:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012992 00000100:00100000:2.0:1767320215.497319:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013248 00000100:00100000:2.0:1767320215.497322:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013248 00000100:00100000:2.0:1767320215.497333:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013120 00000100:00100000:2.0:1767320215.497335:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013120 00000100:00100000:2.0:1767320215.497368:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983012864 00000100:00100000:2.0:1767320215.497372:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983012864 00000100:00100000:2.0:1767320215.497405:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1580 00000100:00100000:2.0:1767320215.497413:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983013120:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320215.497482:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983013120:12345-0@lo:400:kworker.0 Request processed in 70us (4448us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320215.497490:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1580 00000100:00100000:2.0:1767320215.497506:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1581 00000100:00100000:2.0:1767320215.497513:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983012864:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320215.497553:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983012864:12345-0@lo:400:kworker.0 Request processed in 41us (4460us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320215.497559:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1581 00000100:00100000:2.0:1767320215.497571:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088 00000100:00100000:2.0:1767320215.497574:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983012992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320215.497594:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320215.497629:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983012992:12345-0@lo:38:kworker.0 Request processed in 53us (4767us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320215.497635:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088 00000100:00100000:2.0:1767320215.497645:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2089 00000100:00100000:2.0:1767320215.497649:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983013248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320215.497658:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320215.497682:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983013248:12345-0@lo:38:kworker.0 Request processed in 31us (4741us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320215.497688:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2089 00000100:00080000:2.0:1767320215.497707:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320215.497848:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013504 00000100:00100000:2.0:1767320215.497853:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013504 00000100:00100000:2.0:1767320215.497886:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981 00000100:00100000:2.0:1767320215.497892:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983013504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320215.497927:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983013504:12345-0@lo:400:kworker.0 Request processed in 35us (4815us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320215.497934:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981 00000100:00080000:2.0:1767320215.497947:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320215.497995:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320215.498042:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320215.498049:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320215.498054:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983013248:0@lo:38:kworker.0 00000100:00080000:2.0:1767320215.498064:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320215.498067:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320215.498069:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983012992:0@lo:38:kworker.0 00000100:00100000:2.0:1767320215.498098:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013760 00000100:00100000:2.0:1767320215.498102:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013760 00000100:00100000:2.0:1767320215.498119:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2090 00000100:00100000:2.0:1767320215.498123:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983013760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320215.498133:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320215.498162:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983013760:12345-0@lo:38:kworker.0 Request processed in 37us (167us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320215.498168:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2090 00000100:00080000:2.0:1767320215.498190:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320215.498194:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320215.498197:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983013760:0@lo:38:kworker.0 00000100:00100000:3.0:1767320215.514066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983013504:0@lo:400:kworker.0 00000100:00100000:2.0:1767320215.514096:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320215.514164:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013632 00000100:00100000:2.0:1767320215.514171:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013632 00000100:00100000:2.0:1767320215.514229:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 982 00000100:00100000:2.0:1767320215.514239:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983013632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320215.514355:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983013632:12345-0@lo:400:kworker.0 Request processed in 117us (261us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320215.514363:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 982 00000100:00080000:2.0:1767320215.514395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320215.514422:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983013632:0@lo:400:kworker.0 00010000:00080000:3.0:1767320215.514884:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911180 x1853163983013120/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320231 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320215.514896:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983013120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320215.515149:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983013376:12345-0@lo:38:kworker.0 Request processed in 21965us (22141us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320215.515159:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 547 00000100:00080000:3.0:1767320215.515437:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320215.515444:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320215.515448:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983013376:0@lo:38:kworker.0 00010000:00080000:0.0:1767320215.521168:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911500 x1853163983012864/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320231 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320215.521188:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983012864:0@lo:400:kworker.0 00010000:00010000:2.0:1767320215.685078:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320216.708565:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320217.029762:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983013888:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320217.029849:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320217.030302:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983013888 00000100:00100000:3.0:1767320217.030312:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983013888 00000100:00100000:3.0:1767320217.030390:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062 00000100:00100000:3.0:1767320217.030401:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983013888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320217.030431:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320217.030478:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983013888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (628us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320217.030485:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062 00000100:00100000:1.0:1767320217.030988:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983013888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320217.031055:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320217.031065:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983014016:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320217.031102:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320217.031236:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320217.031617:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014016 00000100:00100000:1.0:1767320217.031622:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014016 00000100:00100000:1.0:1767320217.031682:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1063 00000100:00100000:1.0:1767320217.031690:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983014016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320217.031711:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320217.031757:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983014016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (654us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320217.031764:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1063 00000100:00100000:1.0:1767320217.031799:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983014016:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320217.032010:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320217.483575:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320217.483620:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320217.483710:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668864 00000100:00100000:3.0:1767320217.483720:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668864 00000100:00100000:3.0:1767320217.483765:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915668992 00000100:00100000:3.0:1767320217.483769:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915668992 00000100:00100000:0.0:1767320217.483816:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2091 00000100:00080000:1.0:1767320217.483849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320217.483883:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915668864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320217.483907:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320217.483980:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915668864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 97us (414us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320217.484534:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1582 00000100:00100000:3.0:1767320217.484545:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915668992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320217.484588:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915668992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 46us (967us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320217.484595:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1582 00000100:00100000:0.0:1767320217.491992:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2091 00010000:00010000:2.0:1767320217.736383:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320218.756608:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320219.780578:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320220.612473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320220.612481:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320220.612536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320220.612539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320220.612543:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320220.612549:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5464 00000100:00080000:0.0:1767320220.612552:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320220.612557:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320220.612585:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595180 x1853163983014272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320220.612624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320220.612628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320220.612630:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320220.612633:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5464 00000100:00080000:0.0:1767320220.612636:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320220.612640:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320220.612652:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594e00 x1853163983014528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320220.612660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320220.612664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320220.612666:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320220.612669:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5464 00000100:00080000:0.0:1767320220.612672:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320220.612676:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320220.612686:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594380 x1853163983014656/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320220.612690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983014144:0@lo:400:kworker.0 00000100:00080000:0.0:1767320220.612714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320220.612718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320220.612720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320220.612736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5464 00000100:00080000:0.0:1767320220.612738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320220.612741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320220.612753:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163983015040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320220.612766:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320220.612800:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014144 00000100:00100000:0.0:1767320220.612807:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014144 00000100:00100000:1.0:1767320220.612821:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320220.612829:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983014912:0@lo:400:kworker.0 00000100:00100000:1.0:1767320220.612857:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320220.612864:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1583 00000100:00100000:0.0:1767320220.612876:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983014144:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320220.612889:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983014784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320220.612937:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983014144:12345-0@lo:400:kworker.0 Request processed in 63us (171us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320220.612938:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320220.612947:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1583 00000100:00100000:1.0:1767320220.612965:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:0.0:1767320220.612966:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320220.612975:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983014400:0@lo:400:kworker.0 00010000:00080000:0.0:1767320220.612991:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594a80 x1853163983014144/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320236 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320220.613002:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320220.613012:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983014144:0@lo:400:kworker.0 00000100:00100000:0.0:1767320220.613035:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014400 00000100:00100000:0.0:1767320220.613038:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014400 00000100:00100000:0.0:1767320220.613060:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1584 00000100:00100000:0.0:1767320220.613065:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983014400:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320220.613067:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320220.613091:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983014400:12345-0@lo:400:kworker.0 Request processed in 26us (93us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320220.613096:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1584 00010000:00080000:1.0:1767320220.613112:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596d80 x1853163983014400/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320236 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320220.613124:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983014400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320220.613158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983014272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320220.613184:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320220.613199:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983014528:0@lo:38:kworker.0 00000100:00100000:1.0:1767320220.613218:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320220.613234:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983014656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320220.613254:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320220.613305:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983015040:0@lo:38:kworker.0 00000100:00100000:1.0:1767320220.613327:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320220.613355:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014272 00000100:00100000:1.0:1767320220.613360:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014272 00000100:00100000:0.0:1767320220.613364:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014528 00000100:00100000:0.0:1767320220.613368:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014528 00000100:00100000:1.0:1767320220.613380:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015040 00000100:00100000:0.0:1767320220.613385:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2092 00000100:00100000:1.0:1767320220.613385:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015040 00000100:00100000:0.0:1767320220.613390:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983014272:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320220.613405:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2093 00010000:00020000:0.0:1767320220.613406:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320220.613416:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983014528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320220.613429:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320220.613437:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983014272:12345-0@lo:38:kworker.0 Request processed in 47us (252us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320220.613442:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2092 00000100:00100000:0.0:1767320220.613451:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094 00000100:00100000:0.0:1767320220.613454:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983015040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320220.613461:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320220.613479:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983014528:12345-0@lo:38:kworker.0 Request processed in 62us (258us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320220.613486:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983015040:12345-0@lo:38:kworker.0 Request processed in 28us (156us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320220.613492:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2093 00000100:00100000:1.0:1767320220.613513:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014656 00000100:00100000:1.0:1767320220.613517:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014656 00000100:00100000:1.0:1767320220.613540:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 548 00000100:00100000:1.0:1767320220.613547:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983014656:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320220.613550:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014912 00000100:00100000:2.0:1767320220.613554:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014912 00010000:00020000:1.0:1767320220.613558:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320220.613588:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983014656:12345-0@lo:38:kworker.0 Request processed in 39us (331us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320220.613594:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 548 00000100:00100000:2.0:1767320220.613611:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983014784 00000100:00100000:2.0:1767320220.613616:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983014784 00000100:00080000:1.0:1767320220.613639:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320220.613646:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983 00000100:00080000:1.0:1767320220.613647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320220.613654:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983014272:0@lo:38:kworker.0 00000100:00100000:2.0:1767320220.613660:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983014912:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320220.613668:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320220.613675:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320220.613679:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983014528:0@lo:38:kworker.0 00000100:00080000:1.0:1767320220.613690:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320220.613694:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320220.613698:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983014656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320220.613706:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983014912:12345-0@lo:400:kworker.0 Request processed in 49us (847us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320220.613709:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320220.613714:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983 00000100:00080000:1.0:1767320220.613716:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320220.613721:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983015040:0@lo:38:kworker.0 00000100:00100000:2.0:1767320220.613747:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984 00000100:00100000:2.0:1767320220.613755:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983014784:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320220.613792:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983014784:12345-0@lo:400:kworker.0 Request processed in 39us (856us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320220.613799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320220.613800:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984 00000100:00080000:1.0:1767320220.613805:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320220.613834:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983014912:0@lo:400:kworker.0 00000100:00100000:2.0:1767320220.613841:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983014784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320220.614142:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094 00010000:00010000:2.0:1767320220.804765:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320221.828788:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320222.148596:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320222.148621:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983015168:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320222.148691:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320222.148872:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015168 00000100:00100000:1.0:1767320222.148881:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015168 00000100:00100000:1.0:1767320222.148955:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064 00000100:00100000:1.0:1767320222.148966:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983015168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320222.148998:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320222.149046:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983015168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (354us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320222.149054:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064 00000100:00080000:0.0:1767320222.149374:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320222.149428:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983015296:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320222.149457:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320222.149517:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983015168:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320222.149650:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015296 00000100:00100000:1.0:1767320222.149655:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015296 00000100:00100000:1.0:1767320222.149694:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065 00000100:00100000:1.0:1767320222.149702:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983015296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320222.149720:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320222.149804:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320222.149886:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983015296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 184us (427us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320222.149893:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065 00000100:00100000:3.0:1767320222.150193:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983015296:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320222.617609:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320222.617677:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320222.618571:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669248 00000100:00100000:3.0:1767320222.618583:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669248 00000100:00080000:0.0:1767320222.618646:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320222.618663:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669120 00000100:00100000:3.0:1767320222.618668:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669120 00000100:00100000:3.0:1767320222.618696:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1585 00000100:00100000:3.0:1767320222.618708:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915669248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320222.618719:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2095 00000100:00100000:0.0:1767320222.618727:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915669120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320222.618747:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320222.618789:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915669248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (1189us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320222.618796:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915669120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 67us (1117us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320222.618803:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1585 00000100:00100000:0.0:1767320222.618883:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2095 00010000:00010000:1.0:1767320222.852468:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320223.881005:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320224.902886:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320225.732735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320225.732746:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320225.732799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320225.732803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320225.732806:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320225.732813:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5469 00000100:00080000:3.0:1767320225.732817:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320225.732823:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320225.732844:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910a80 x1853163983015552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320225.732888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320225.732892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320225.732894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320225.732898:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5469 00000100:00080000:3.0:1767320225.732901:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320225.732905:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320225.732919:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983015424:0@lo:400:kworker.0 00000100:00100000:3.0:1767320225.732920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983015808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320225.732940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320225.732943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320225.732945:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320225.732948:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5469 00000100:00080000:3.0:1767320225.732951:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320225.732955:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320225.732967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910380 x1853163983015936/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320225.732983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320225.732986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320225.732988:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320225.732991:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5469 00000100:00080000:3.0:1767320225.732992:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320225.732997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320225.733002:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320225.733009:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912300 x1853163983016320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320225.733028:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983015680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320225.733041:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320225.733050:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983016192:0@lo:400:kworker.0 00000100:00100000:3.0:1767320225.733056:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320225.733079:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320225.733086:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983016064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320225.733087:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320225.733114:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320225.733130:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016192 00000100:00100000:2.0:1767320225.733137:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016192 00000100:00100000:3.0:1767320225.733158:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015424 00000100:00100000:3.0:1767320225.733163:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015424 00000100:00100000:2.0:1767320225.733197:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016064 00000100:00100000:2.0:1767320225.733201:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016064 00000100:00100000:3.0:1767320225.733214:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015680 00000100:00100000:3.0:1767320225.733225:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015680 00000100:00100000:2.0:1767320225.733229:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 985 00000100:00100000:2.0:1767320225.733241:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983016192:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320225.733254:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1586 00000100:00100000:3.0:1767320225.733262:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983015424:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320225.733313:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983016192:12345-0@lo:400:kworker.0 Request processed in 74us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320225.733317:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983015424:12345-0@lo:400:kworker.0 Request processed in 55us (320us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320225.733329:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 985 00000100:00100000:3.0:1767320225.733329:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1586 00000100:00100000:2.0:1767320225.733360:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986 00000100:00100000:3.0:1767320225.733361:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1587 00000100:00100000:2.0:1767320225.733372:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983016064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320225.733375:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983015680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320225.733411:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983016064:12345-0@lo:400:kworker.0 Request processed in 41us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320225.733421:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986 00000100:00100000:3.0:1767320225.733422:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983015680:12345-0@lo:400:kworker.0 Request processed in 47us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320225.733430:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1587 00000100:00100000:2.0:1767320225.733457:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983016192:0@lo:400:kworker.0 00010000:00080000:2.0:1767320225.733473:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911c00 x1853163983015424/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320241 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320225.733480:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983016064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320225.733489:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983015424:0@lo:400:kworker.0 00010000:00080000:3.0:1767320225.733497:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911880 x1853163983015680/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320241 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320225.733509:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983015680:0@lo:400:kworker.0 00000100:00100000:1.0:1767320225.733713:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983015552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320225.733741:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320225.733769:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983015808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320225.733787:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320225.733809:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983015936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320225.733832:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320225.733867:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983016320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320225.733888:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320225.733921:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015936 00000100:00100000:1.0:1767320225.733925:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015936 00000100:00100000:1.0:1767320225.733945:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 549 00000100:00100000:1.0:1767320225.733950:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983015936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320225.733965:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320225.733990:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983015936:12345-0@lo:38:kworker.0 Request processed in 40us (157us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320225.733994:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 549 00000100:00080000:1.0:1767320225.734015:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320225.734018:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320225.734020:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983015936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320225.734060:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015552 00000100:00100000:1.0:1767320225.734062:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015552 00000100:00100000:1.0:1767320225.734072:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983015808 00000100:00100000:1.0:1767320225.734073:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983015808 00000100:00100000:1.0:1767320225.734078:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016320 00000100:00100000:1.0:1767320225.734079:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016320 00000100:00100000:1.0:1767320225.734084:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2096 00000100:00100000:1.0:1767320225.734087:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983015552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320225.734094:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320225.734124:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983015552:12345-0@lo:38:kworker.0 Request processed in 36us (383us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320225.734129:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2096 00000100:00100000:1.0:1767320225.734139:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097 00000100:00100000:1.0:1767320225.734142:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983015808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320225.734150:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320225.734172:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983015808:12345-0@lo:38:kworker.0 Request processed in 28us (384us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320225.734178:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097 00000100:00100000:1.0:1767320225.734203:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2098 00000100:00100000:1.0:1767320225.734216:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983016320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320225.734223:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320225.734241:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983016320:12345-0@lo:38:kworker.0 Request processed in 33us (353us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320225.734246:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2098 00000100:00080000:1.0:1767320225.734292:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320225.734295:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320225.734297:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983015552:0@lo:38:kworker.0 00000100:00080000:1.0:1767320225.734302:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320225.734304:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320225.734306:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983015808:0@lo:38:kworker.0 00000100:00080000:1.0:1767320225.734310:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320225.734312:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320225.734313:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983016320:0@lo:38:kworker.0 00000100:00080000:0.0:1767320225.734837:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320225.734843:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320225.734845:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320225.924651:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320226.948559:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320227.275561:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983016448:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320227.275635:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320227.275900:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016448 00000100:00100000:0.0:1767320227.275910:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016448 00000100:00100000:0.0:1767320227.276024:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1066 00000100:00100000:0.0:1767320227.276034:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983016448:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320227.276068:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320227.276129:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983016448:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (494us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320227.276137:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1066 00000100:00080000:2.0:1767320227.276193:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320227.276262:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983016448:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320227.276355:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983016576:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320227.276391:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320227.287423:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016576 00000100:00100000:3.0:1767320227.287448:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016576 00000100:00100000:3.0:1767320227.287522:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067 00000100:00100000:3.0:1767320227.287535:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983016576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320227.287571:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320227.287677:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983016576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 147us (11285us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320227.287684:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067 00000100:00100000:3.0:1767320227.287767:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983016576:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320227.291360:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320227.727264:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320227.727442:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669504 00000100:00100000:3.0:1767320227.727452:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669504 00000100:00100000:3.0:1767320227.727590:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1588 00000100:00080000:2.0:1767320227.727591:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320227.727603:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915669504:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320227.727694:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915669504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320227.727701:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1588 00000100:00100000:0.0:1767320227.729133:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320227.729378:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669376 00000100:00100000:3.0:1767320227.729386:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669376 00000100:00100000:3.0:1767320227.729712:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2099 00000100:00100000:3.0:1767320227.729721:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915669376:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320227.729756:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320227.729812:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915669376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (682us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320227.729819:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2099 00010000:00010000:2.0:1767320227.976306:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320228.997099:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320230.022911:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320230.855451:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320230.855465:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320230.855541:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320230.855547:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320230.855552:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320230.855577:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5474 00000100:00080000:3.0:1767320230.855582:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320230.855590:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320230.855615:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911500 x1853163983016832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320230.855665:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320230.855669:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320230.855671:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320230.855675:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5474 00000100:00080000:3.0:1767320230.855678:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320230.855682:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320230.855695:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983017088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320230.855705:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320230.855708:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320230.855710:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320230.855714:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5474 00000100:00080000:3.0:1767320230.855716:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320230.855719:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320230.855730:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910a80 x1853163983017216/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320230.855748:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320230.855751:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320230.855752:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320230.855755:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5474 00000100:00080000:3.0:1767320230.855757:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320230.855760:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320230.855768:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910e00 x1853163983017600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320230.855794:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983016960:0@lo:400:kworker.0 00000100:00100000:3.0:1767320230.855861:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320230.855932:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320230.855939:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983017344:0@lo:400:kworker.0 00000100:00100000:3.0:1767320230.855979:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320230.856013:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016960 00000100:00100000:3.0:1767320230.856020:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016960 00000100:00100000:3.0:1767320230.856073:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1589 00000100:00100000:3.0:1767320230.856084:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983016960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320230.856147:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983016960:12345-0@lo:400:kworker.0 Request processed in 65us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320230.856154:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1589 00000100:00100000:3.0:1767320230.856180:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017344 00000100:00100000:3.0:1767320230.856184:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017344 00000100:00100000:3.0:1767320230.856243:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987 00000100:00100000:3.0:1767320230.856250:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983017344:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320230.856353:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983017344:12345-0@lo:400:kworker.0 Request processed in 104us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320230.856371:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987 00000100:00080000:3.0:1767320230.856389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320230.856394:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320230.856424:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983017344:0@lo:400:kworker.0 00010000:00080000:3.0:1767320230.856435:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912300 x1853163983016960/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320246 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320230.856444:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983016960:0@lo:400:kworker.0 00000100:00100000:2.0:1767320230.856745:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983017472:0@lo:400:kworker.0 00000100:00100000:2.0:1767320230.856797:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320230.856860:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320230.856866:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983016704:0@lo:400:kworker.0 00000100:00100000:2.0:1767320230.856903:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320230.856992:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016704 00000100:00100000:2.0:1767320230.857012:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016704 00000100:00100000:2.0:1767320230.857058:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1590 00000100:00100000:2.0:1767320230.857081:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983016704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320230.857141:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983016704:12345-0@lo:400:kworker.0 Request processed in 76us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320230.857147:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1590 00000100:00080000:2.0:1767320230.857175:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320230.857209:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911880 x1853163983016704/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320246 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320230.857219:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983016704:0@lo:400:kworker.0 00000100:00100000:0.0:1767320230.857464:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983016832:0@lo:38:kworker.0 00000100:00100000:0.0:1767320230.857516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320230.857577:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983017088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320230.857616:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320230.857652:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983017216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320230.857676:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320230.857696:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983017600:0@lo:38:kworker.0 00000100:00100000:0.0:1767320230.857716:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320230.857741:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983016832 00000100:00100000:0.0:1767320230.857745:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983016832 00000100:00100000:0.0:1767320230.857760:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017088 00000100:00100000:0.0:1767320230.857762:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017088 00000100:00100000:0.0:1767320230.857770:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017600 00000100:00100000:0.0:1767320230.857772:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017600 00000100:00100000:0.0:1767320230.857781:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100 00000100:00100000:0.0:1767320230.857786:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983016832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320230.857804:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320230.857836:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983016832:12345-0@lo:38:kworker.0 Request processed in 49us (320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320230.857843:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100 00000100:00100000:0.0:1767320230.857857:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2101 00000100:00100000:0.0:1767320230.857861:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983017088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320230.857870:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320230.857894:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983017088:12345-0@lo:38:kworker.0 Request processed in 31us (277us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320230.857900:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2101 00000100:00100000:0.0:1767320230.857911:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2102 00000100:00100000:0.0:1767320230.857915:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983017600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320230.857922:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320230.857942:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983017600:12345-0@lo:38:kworker.0 Request processed in 26us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320230.857948:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2102 00000100:00080000:0.0:1767320230.858000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320230.858006:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320230.858009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983016832:0@lo:38:kworker.0 00000100:00080000:0.0:1767320230.858034:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320230.858037:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320230.858040:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983017088:0@lo:38:kworker.0 00000100:00080000:0.0:1767320230.858048:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320230.858051:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320230.858054:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983017600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320230.859650:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017472 00000100:00100000:1.0:1767320230.859662:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017472 00000100:00100000:1.0:1767320230.859793:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988 00000100:00100000:1.0:1767320230.859823:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983017472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320230.859960:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983017472:12345-0@lo:400:kworker.0 Request processed in 149us (3152us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320230.860574:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320230.860629:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017216 00000100:00100000:2.0:1767320230.860651:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017216 00000100:00100000:2.0:1767320230.860709:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 550 00000100:00100000:2.0:1767320230.860714:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983017216:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320230.860758:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320230.860834:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983017216:12345-0@lo:38:kworker.0 Request processed in 117us (3155us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320230.860841:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 550 00000100:00100000:2.0:1767320230.860946:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983017472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320230.864988:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988 00000100:00080000:0.0:1767320230.865221:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320230.865229:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320230.865233:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983017216:0@lo:38:kworker.0 00010000:00010000:2.0:1767320231.044824:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320232.068497:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320232.389590:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320232.389615:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983017728:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320232.389684:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320232.390015:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017728 00000100:00100000:0.0:1767320232.390023:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017728 00000100:00100000:0.0:1767320232.390103:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068 00000100:00100000:0.0:1767320232.390113:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983017728:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320232.390143:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320232.390194:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983017728:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (510us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320232.390312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320232.390362:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983017728:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320232.390764:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068 00000100:00100000:2.0:1767320232.391361:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320232.391375:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983017856:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320232.391425:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320232.391579:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017856 00000100:00100000:0.0:1767320232.391586:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017856 00000100:00100000:0.0:1767320232.391635:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1069 00000100:00100000:0.0:1767320232.391643:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983017856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320232.391665:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320232.391702:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983017856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320232.391708:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1069 00000100:00100000:0.0:1767320232.391738:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983017856:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320232.391952:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320232.844976:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320232.845215:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669760 00000100:00100000:1.0:1767320232.845227:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669760 00000100:00100000:1.0:1767320232.845342:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1591 00000100:00100000:1.0:1767320232.845353:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915669760:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320232.845446:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915669760:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (494us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320232.845454:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1591 00000100:00080000:2.0:1767320232.845603:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320232.849974:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320232.850116:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915669632 00000100:00100000:3.0:1767320232.850126:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915669632 00000100:00100000:3.0:1767320232.850182:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2103 00000100:00100000:3.0:1767320232.850190:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915669632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320232.850213:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320232.850315:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915669632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 123us (363us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320232.850322:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2103 00010000:00010000:2.0:1767320233.092374:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320234.116519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320235.140589:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320235.976071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320235.976090:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320235.976186:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983017984:0@lo:400:kworker.0 00000100:00100000:2.0:1767320235.976258:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320235.976496:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983017984 00000100:00100000:1.0:1767320235.976503:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983017984 00000100:00100000:1.0:1767320235.976569:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1592 00000100:00080000:3.0:1767320235.976573:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320235.976580:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983017984:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320235.976662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320235.976667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320235.976671:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320235.976676:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5479 00000100:00080000:0.0:1767320235.976680:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320235.976684:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320235.976705:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52680 x1853163983018112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320235.976740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320235.976744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320235.976745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320235.976755:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5479 00000100:00080000:0.0:1767320235.976758:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320235.976761:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320235.976771:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53800 x1853163983018368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320235.976780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320235.976783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320235.976785:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320235.976789:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5479 00000100:00080000:0.0:1767320235.976791:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320235.976796:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320235.976809:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983018240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320235.976810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51180 x1853163983018496/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320235.976849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320235.976872:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320235.976879:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983018624:0@lo:400:kworker.0 00000100:00100000:1.0:1767320235.976893:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983017984:12345-0@lo:400:kworker.0 Request processed in 314us (632us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320235.976902:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1592 00000100:00100000:3.0:1767320235.976907:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320235.976936:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018624 00000100:00100000:3.0:1767320235.976940:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018624 00000100:00100000:3.0:1767320235.976973:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989 00000100:00100000:3.0:1767320235.976979:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983018624:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320235.977023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767320235.977025:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983018624:12345-0@lo:400:kworker.0 Request processed in 45us (119us total) trans 0 rc 0/0 00000100:00080000:0.0:1767320235.977029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320235.977033:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320235.977040:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5479 00000100:00080000:0.0:1767320235.977043:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320235.977046:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320235.977059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967800 x1853163983018880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320235.977094:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983018112:0@lo:38:kworker.0 00000100:00100000:3.0:1767320235.977117:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989 00000100:00100000:0.0:1767320235.977123:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320235.977158:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018240 00000100:00100000:0.0:1767320235.977159:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983018368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320235.977163:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018240 00000100:00100000:0.0:1767320235.977185:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320235.977193:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018112 00000100:00100000:3.0:1767320235.977198:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018112 00000100:00100000:3.0:1767320235.977211:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018368 00000100:00100000:3.0:1767320235.977214:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018368 00000100:00100000:0.0:1767320235.977215:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983018496:0@lo:38:kworker.0 00000100:00100000:3.0:1767320235.977227:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1593 00000100:00100000:3.0:1767320235.977236:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983018240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320235.977244:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320235.977267:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983018880:0@lo:38:kworker.0 00000100:00100000:3.0:1767320235.977298:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983018240:12345-0@lo:400:kworker.0 Request processed in 63us (449us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320235.977307:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1593 00000100:00100000:0.0:1767320235.977319:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320235.977323:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2104 00000100:00100000:3.0:1767320235.977329:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983018112:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320235.977346:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320235.977348:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018880 00000100:00100000:0.0:1767320235.977352:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018880 00000100:00100000:0.0:1767320235.977367:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2105 00000100:00100000:0.0:1767320235.977373:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983018368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320235.977385:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320235.977393:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983018112:12345-0@lo:38:kworker.0 Request processed in 62us (272us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320235.977400:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2104 00000100:00100000:0.0:1767320235.977417:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983018368:12345-0@lo:38:kworker.0 Request processed in 40us (230us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320235.977418:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106 00000100:00100000:3.0:1767320235.977427:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983018880:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320235.977429:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2105 00010000:00020000:3.0:1767320235.977441:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320235.977475:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983018752:0@lo:400:kworker.0 00000100:00100000:0.0:1767320235.977495:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320235.977515:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018752 00000100:00100000:0.0:1767320235.977518:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018752 00000100:00100000:0.0:1767320235.977538:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990 00000100:00100000:0.0:1767320235.977543:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983018752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320235.977570:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983018752:12345-0@lo:400:kworker.0 Request processed in 27us (73us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320235.977575:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990 00000100:00080000:0.0:1767320235.977587:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320235.977589:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320235.977591:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320235.977611:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983018752:0@lo:400:kworker.0 00000100:00080000:0.0:1767320235.977630:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320235.977634:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320235.977637:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983018112:0@lo:38:kworker.0 00000100:00080000:0.0:1767320235.977644:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320235.977647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320235.977650:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983018368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320235.977674:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983018496 00000100:00100000:0.0:1767320235.977677:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983018496 00000100:00100000:0.0:1767320235.977693:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 551 00000100:00100000:0.0:1767320235.977697:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983018496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320235.977706:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320235.977729:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983018496:12345-0@lo:38:kworker.0 Request processed in 31us (489us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320235.977735:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 551 00000100:00080000:0.0:1767320235.977754:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320235.977757:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320235.977760:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983018496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320235.977782:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983018624:0@lo:400:kworker.0 00010000:00080000:0.0:1767320235.977791:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f50700 x1853163983018240/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320251 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320235.977800:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983018240:0@lo:400:kworker.0 00010000:00080000:2.0:1767320235.978462:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596a00 x1853163983017984/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320251 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320235.978474:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983017984:0@lo:400:kworker.0 00000100:00100000:3.0:1767320235.980169:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983018880:12345-0@lo:38:kworker.0 Request processed in 2734us (2855us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320235.980181:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106 00000100:00080000:0.0:1767320235.982926:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320235.982937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320235.982942:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983018880:0@lo:38:kworker.0 00010000:00010000:2.0:1767320236.164814:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320236.943503:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767320236.943539:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915669888 offset 0 length 520: 4 00000100:00100000:0.0:1767320236.943567:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320236.943833:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670144 00000100:00100000:1.0:1767320236.943844:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670144 00000100:00100000:3.0:1767320236.943902:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670016 00000100:00100000:3.0:1767320236.943907:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670016 00000100:00100000:3.0:1767320236.943939:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1594 00000100:00100000:3.0:1767320236.943949:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915670144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767320236.944046:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320236.944049:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915670144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (556us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320236.944079:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2107 00000100:00100000:1.0:1767320236.944087:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670016:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320236.944108:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320236.944150:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 63us (582us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320236.944158:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2107 00000100:00100000:3.0:1767320236.957461:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1594 00010000:00010000:2.0:1767320237.188489:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320237.511102:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983019008:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320237.511184:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320237.511821:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019008 00000100:00100000:0.0:1767320237.511830:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019008 00000100:00100000:0.0:1767320237.511902:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070 00000100:00100000:0.0:1767320237.511913:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983019008:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320237.511952:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320237.512011:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983019008:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (826us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320237.512019:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070 00000100:00100000:0.0:1767320237.512067:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983019008:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320237.512158:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320237.512692:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320237.512704:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983019136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320237.512741:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320237.512897:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019136 00000100:00100000:0.0:1767320237.512903:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019136 00000100:00080000:1.0:1767320237.513001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320237.513037:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1767320237.513045:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983019136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320237.513063:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320237.513128:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983019136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (377us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320237.513136:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071 00000100:00100000:3.0:1767320237.513198:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983019136:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320238.212367:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320239.238178:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320240.068889:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320215/real 1767320215] req@ffff93460a912a00 x1853163983012736/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320240 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320240.068957:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a912a00 x1853163983012736/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320240 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320240.068977:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320240.069052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320240.069057:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983012736:0@lo:250:kworker.0 00000100:00080000:3.0:1767320240.077090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320240.077100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320240.077104:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320240.077111:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5464 00000100:00080000:3.0:1767320240.077117:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320240.077124:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320240.077289:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34380 x1853163983019264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320240.077346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320240.077350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320240.077352:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320240.077356:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5484 00000100:00080000:3.0:1767320240.077358:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320240.077362:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320240.077588:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983019264:0@lo:250:kworker.0 00000400:00000100:0.0:1767320240.077733:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983019264 offset 0 length 520: 4 00000100:00100000:1.0:1767320240.077847:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983019392:0@lo:400:kworker.0 00000100:00100000:1.0:1767320240.077882:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320240.077986:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019392 00000100:00100000:0.0:1767320240.077994:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019392 00000100:00100000:0.0:1767320240.078048:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1595 00000100:00100000:0.0:1767320240.078058:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983019392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320240.078098:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983019392:12345-0@lo:400:kworker.0 Request processed in 43us (216us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320240.078105:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1595 00000100:00080000:1.0:1767320240.078286:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320240.078313:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34000 x1853163983019392/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320256 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320240.078325:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983019392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320240.078572:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163983019520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320240.078626:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983019648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320240.078667:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320240.078770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320240.078774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320240.078776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320240.078780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5484 00000100:00080000:3.0:1767320240.078783:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320240.078787:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320240.078801:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163983019776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320240.078810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320240.078813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320240.078815:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320240.078818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5484 00000100:00080000:3.0:1767320240.078820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320240.078823:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320240.078834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163983019904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320240.078883:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983020032:0@lo:400:kworker.0 00000100:00100000:0.0:1767320240.078909:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320240.079010:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020032 00000100:00100000:0.0:1767320240.079015:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020032 00000100:00100000:0.0:1767320240.079045:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767320240.079051:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983020032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320240.079082:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983020032:12345-0@lo:400:kworker.0 Request processed in 32us (172us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320240.079088:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 991 00000100:00100000:0.0:1767320240.079114:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983020032:0@lo:400:kworker.0 00000100:00100000:0.0:1767320240.079127:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019648 00000100:00100000:0.0:1767320240.079130:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019648 00000100:00100000:0.0:1767320240.079150:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596 00000100:00100000:0.0:1767320240.079168:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983019648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320240.079267:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983019520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320240.079310:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320240.079325:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983019776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320240.079342:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320240.079355:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983019904:0@lo:38:kworker.0 00000100:00100000:2.0:1767320240.079389:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320240.079413:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019520 00000100:00100000:2.0:1767320240.079417:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019520 00000100:00100000:2.0:1767320240.079432:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019776 00000100:00100000:2.0:1767320240.079434:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019776 00000100:00100000:2.0:1767320240.079442:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2108 00000100:00100000:2.0:1767320240.079447:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983019520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320240.079463:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320240.079502:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983019520:12345-0@lo:38:kworker.0 Request processed in 54us (190us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320240.079508:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2108 00000100:00100000:2.0:1767320240.079520:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109 00000100:00100000:2.0:1767320240.079524:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983019776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320240.079532:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320240.079554:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983019776:12345-0@lo:38:kworker.0 Request processed in 29us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320240.079561:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109 00000100:00080000:2.0:1767320240.079582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320240.079586:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320240.079589:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983019520:0@lo:38:kworker.0 00000100:00080000:2.0:1767320240.079596:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320240.079599:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320240.079602:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983019776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320240.079630:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983019904 00000100:00100000:2.0:1767320240.079633:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983019904 00000100:00100000:2.0:1767320240.080346:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 552 00000100:00100000:2.0:1767320240.080351:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983019904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320240.080362:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320240.080415:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983019904:12345-0@lo:38:kworker.0 Request processed in 61us (1040us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320240.080421:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 552 00000100:00080000:2.0:1767320240.080449:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320240.080453:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320240.080455:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983019904:0@lo:38:kworker.0 00000100:00080000:2.0:1767320240.080509:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320240.080513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320240.092012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320240.092019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320240.092023:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320240.092031:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5484 00000100:00080000:2.0:1767320240.092036:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320240.092042:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320240.092066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91f80 x1853163983020288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320240.092112:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983020288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320240.092191:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320240.092254:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320240.092261:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983020160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320240.092314:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320240.092355:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020160 00000100:00100000:2.0:1767320240.092362:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020160 00000100:00100000:2.0:1767320240.093103:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992 00000100:00100000:2.0:1767320240.093112:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983020160:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320240.093175:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983020160:12345-0@lo:400:kworker.0 Request processed in 64us (859us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320240.093182:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992 00000100:00080000:2.0:1767320240.093205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320240.093232:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983020160:0@lo:400:kworker.0 00000100:00100000:0.0:1767320240.093411:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983019648:12345-0@lo:400:kworker.0 Request processed in 14242us (14742us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320240.093420:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596 00000100:00100000:0.0:1767320240.093437:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020288 00000100:00100000:0.0:1767320240.093440:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020288 00000100:00100000:0.0:1767320240.093455:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2110 00000100:00100000:0.0:1767320240.093459:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983020288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320240.093479:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320240.093520:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983020288:12345-0@lo:38:kworker.0 Request processed in 59us (1329us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320240.093526:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2110 00000100:00080000:2.0:1767320240.099112:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320240.099125:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320240.099130:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983020288:0@lo:38:kworker.0 00010000:00080000:1.0:1767320240.102850:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36680 x1853163983019648/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320256 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320240.102869:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983019648:0@lo:400:kworker.0 00010000:00010000:2.0:1767320240.260935:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320241.284606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320242.065548:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320242.065610:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320242.067479:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670400 00000100:00100000:1.0:1767320242.067494:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670400 00000100:00100000:1.0:1767320242.067580:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670272 00000100:00100000:1.0:1767320242.067584:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670272 00000100:00100000:1.0:1767320242.067607:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1597 00000100:00100000:1.0:1767320242.067618:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915670400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320242.067702:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915670400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (2164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320242.067710:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1597 00000100:00100000:1.0:1767320242.067730:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2111 00000100:00100000:1.0:1767320242.067735:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320242.067755:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320242.067781:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (2171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320242.067789:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2111 00000100:00080000:2.0:1767320242.068020:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320242.309037:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320242.628967:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320242.629003:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983020416:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320242.629100:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320242.629227:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020416 00000100:00100000:0.0:1767320242.629239:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020416 00000100:00100000:0.0:1767320242.629360:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1767320242.629371:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983020416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320242.629413:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320242.629480:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983020416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (379us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320242.629487:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1767320242.629549:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983020416:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320242.629755:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320242.631641:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983020544:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320242.631754:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320242.631971:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020544 00000100:00100000:2.0:1767320242.631982:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020544 00000100:00100000:2.0:1767320242.632073:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073 00000100:00100000:2.0:1767320242.632085:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983020544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320242.632121:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320242.632186:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983020544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (432us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320242.632194:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073 00000100:00100000:2.0:1767320242.632253:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983020544:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320242.632610:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320243.343400:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320244.357396:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320245.188633:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320245.188645:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320245.188792:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320245.188796:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320245.188801:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320245.188812:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5488 00000100:00080000:3.0:1767320245.188818:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320245.188856:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320245.188885:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35880 x1853163983020800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320245.188934:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320245.188937:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320245.188939:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320245.188942:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5488 00000100:00080000:3.0:1767320245.188944:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320245.188947:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320245.188957:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163983021056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320245.188966:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320245.188970:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320245.188972:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320245.188975:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5488 00000100:00080000:3.0:1767320245.188977:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320245.188981:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320245.188993:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163983021184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320245.189014:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320245.189017:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320245.189019:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320245.189023:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5488 00000100:00080000:3.0:1767320245.189025:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320245.189028:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320245.189039:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163983021568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320245.189073:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983021312:0@lo:400:kworker.0 00000100:00100000:2.0:1767320245.189192:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983021440:0@lo:400:kworker.0 00000100:00100000:2.0:1767320245.189331:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320245.189575:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983020672:0@lo:400:kworker.0 00000100:00100000:1.0:1767320245.189665:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983020800:0@lo:38:kworker.0 00000100:00100000:2.0:1767320245.189669:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021440 00000100:00100000:2.0:1767320245.189693:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021440 00000100:00100000:2.0:1767320245.189770:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993 00000100:00100000:2.0:1767320245.189781:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983021440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.192999:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.193093:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320245.193108:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983020928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.193167:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.193254:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021312 00000100:00100000:3.0:1767320245.193265:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021312 00000100:00100000:3.0:1767320245.193357:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 994 00000100:00100000:3.0:1767320245.193368:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983021312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.193438:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983021312:12345-0@lo:400:kworker.0 Request processed in 71us (450us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320245.193446:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 994 00000100:00100000:3.0:1767320245.193496:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983021312:0@lo:400:kworker.0 00000100:00080000:3.0:1767320245.193552:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320245.193556:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320245.193587:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020928 00000100:00100000:3.0:1767320245.193590:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020928 00000100:00100000:3.0:1767320245.193626:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1598 00000100:00100000:3.0:1767320245.193632:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983020928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.193666:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983020928:12345-0@lo:400:kworker.0 Request processed in 35us (499us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320245.193673:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1598 00000100:00080000:3.0:1767320245.193695:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320245.193710:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36680 x1853163983020928/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320261 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320245.193723:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983020928:0@lo:400:kworker.0 00000100:00100000:0.0:1767320245.194241:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.194429:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020672 00000100:00100000:3.0:1767320245.194434:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020672 00000100:00100000:3.0:1767320245.194486:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1599 00000100:00100000:3.0:1767320245.194494:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983020672:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.194546:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983020672:12345-0@lo:400:kworker.0 Request processed in 52us (306us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320245.194552:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1599 00000100:00080000:0.0:1767320245.194688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320245.194716:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34e00 x1853163983020672/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320261 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320245.194729:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983020672:0@lo:400:kworker.0 00000100:00100000:2.0:1767320245.195406:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983021440:12345-0@lo:400:kworker.0 Request processed in 5622us (6070us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320245.195419:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993 00000100:00100000:0.0:1767320245.195517:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983021440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320245.195598:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.195646:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983021056:0@lo:38:kworker.0 00000100:00100000:3.0:1767320245.195686:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.195715:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983021184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320245.195742:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320245.195770:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983021568:0@lo:38:kworker.0 00000100:00100000:3.0:1767320245.195792:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320245.195879:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983020800 00000100:00100000:0.0:1767320245.195885:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983020800 00000100:00100000:0.0:1767320245.195919:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021056 00000100:00100000:0.0:1767320245.195922:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021056 00000100:00100000:0.0:1767320245.195933:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021568 00000100:00100000:0.0:1767320245.195936:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021568 00000100:00100000:0.0:1767320245.195947:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767320245.195954:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983020800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320245.195990:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320245.196038:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983020800:12345-0@lo:38:kworker.0 Request processed in 84us (440us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320245.196045:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767320245.196058:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2113 00000100:00100000:0.0:1767320245.196062:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983021056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320245.196071:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320245.196093:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983021056:12345-0@lo:38:kworker.0 Request processed in 30us (407us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320245.196099:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2113 00000100:00100000:0.0:1767320245.196122:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2114 00000100:00100000:0.0:1767320245.196126:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983021568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320245.196135:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320245.196160:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983021568:12345-0@lo:38:kworker.0 Request processed in 32us (366us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320245.196165:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2114 00000100:00100000:2.0:1767320245.198243:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021184 00000100:00100000:2.0:1767320245.198266:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021184 00000100:00100000:2.0:1767320245.198373:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 553 00000100:00100000:2.0:1767320245.198383:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983021184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320245.198411:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320245.198511:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983021184:12345-0@lo:38:kworker.0 Request processed in 127us (2766us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320245.198523:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 553 00000100:00080000:3.0:1767320245.198698:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320245.198705:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320245.198709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983020800:0@lo:38:kworker.0 00000100:00080000:3.0:1767320245.198722:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320245.198726:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320245.198729:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983021056:0@lo:38:kworker.0 00000100:00080000:3.0:1767320245.198738:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320245.198741:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320245.198745:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983021184:0@lo:38:kworker.0 00000100:00080000:3.0:1767320245.198756:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320245.198760:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320245.198762:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983021568:0@lo:38:kworker.0 00010000:00010000:2.0:1767320245.382250:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320246.404557:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320247.180329:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320247.180421:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320247.180785:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670656 00000100:00100000:3.0:1767320247.180809:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670656 00000100:00100000:3.0:1767320247.180885:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670528 00000100:00100000:3.0:1767320247.180889:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670528 00000100:00100000:3.0:1767320247.180913:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1600 00000100:00100000:3.0:1767320247.180923:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915670656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320247.181000:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915670656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (681us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320247.181008:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1600 00000100:00100000:3.0:1767320247.181027:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2115 00000100:00100000:3.0:1767320247.181031:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320247.181050:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320247.181076:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (655us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320247.181083:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2115 00000100:00080000:0.0:1767320247.181524:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320247.428894:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320247.748673:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983021696:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320247.748775:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320247.748871:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320247.748891:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983021824:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320247.748943:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320247.756544:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021696 00000100:00100000:2.0:1767320247.756566:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021696 00000100:00100000:2.0:1767320247.756648:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021824 00000100:00100000:2.0:1767320247.756653:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021824 00000100:00100000:2.0:1767320247.756679:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074 00000100:00100000:2.0:1767320247.756691:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983021696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320247.756735:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320247.756834:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983021696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 145us (8058us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320247.756842:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074 00000100:00100000:2.0:1767320247.756861:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1075 00000100:00100000:2.0:1767320247.756868:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983021824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320247.756885:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320247.756919:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983021824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (7977us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320247.756991:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1075 00000100:00100000:2.0:1767320247.757055:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983021696:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320247.757670:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320247.757677:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320247.757718:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983021824:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320248.456324:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320249.479661:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320250.309024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320250.309037:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320250.309114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320250.309119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320250.309128:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320250.309140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5493 00000100:00080000:3.0:1767320250.309146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320250.309153:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320250.309162:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320250.309180:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983021952:0@lo:400:kworker.0 00000100:00100000:3.0:1767320250.309180:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912300 x1853163983022080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320250.309220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320250.309224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320250.309226:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320250.309231:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5493 00000100:00080000:3.0:1767320250.309233:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320250.309238:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320250.309251:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983022208:0@lo:400:kworker.0 00000100:00100000:3.0:1767320250.309254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163983022336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320250.309265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320250.309289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320250.309292:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320250.309295:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5493 00000100:00100000:1.0:1767320250.309298:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320250.309298:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320250.309304:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320250.309318:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36a00 x1853163983022464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320250.309344:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320250.309349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320250.309350:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983022720:0@lo:400:kworker.0 00000100:00080000:3.0:1767320250.309354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320250.309356:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320250.309361:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5493 00000100:00080000:3.0:1767320250.309364:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320250.309368:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320250.309375:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320250.309383:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163983022848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320250.309386:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983022592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320250.309388:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320250.309423:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983022080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320250.309442:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983021952 00000100:00100000:1.0:1767320250.309451:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983021952 00000100:00100000:0.0:1767320250.309456:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320250.309462:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320250.309513:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022208 00000100:00100000:1.0:1767320250.309517:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022208 00000100:00100000:1.0:1767320250.309551:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022080 00000100:00100000:1.0:1767320250.309555:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022080 00000100:00100000:2.0:1767320250.309903:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1601 00000100:00100000:2.0:1767320250.309920:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983021952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320250.309941:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1602 00000100:00100000:1.0:1767320250.309953:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983022208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320250.318425:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2116 00000100:00100000:0.0:1767320250.318440:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983022080:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320250.318471:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320250.318572:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983022080:12345-0@lo:38:kworker.0 Request processed in 132us (9111us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320250.318580:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2116 00000100:00080000:0.0:1767320250.318653:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320250.318692:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022720 00000100:00100000:0.0:1767320250.318702:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022720 00000100:00100000:0.0:1767320250.318778:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022592 00000100:00100000:0.0:1767320250.318782:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022592 00000100:00100000:0.0:1767320250.318812:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995 00000100:00100000:0.0:1767320250.318820:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983022720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320250.318889:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983022720:12345-0@lo:400:kworker.0 Request processed in 70us (9503us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320250.318897:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995 00000100:00100000:0.0:1767320250.318915:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996 00000100:00100000:0.0:1767320250.318923:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983022592:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320250.318967:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320250.318975:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983022592:12345-0@lo:400:kworker.0 Request processed in 52us (9521us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320250.318978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320250.318986:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983022080:0@lo:38:kworker.0 00000100:00100000:0.0:1767320250.318988:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996 00000100:00100000:3.0:1767320250.319004:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983022336:0@lo:38:kworker.0 00000100:00080000:0.0:1767320250.319011:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320250.319015:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320250.319052:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320250.319059:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983022592:0@lo:400:kworker.0 00000100:00100000:3.0:1767320250.319083:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022336 00000100:00100000:3.0:1767320250.319088:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022336 00000100:00100000:3.0:1767320250.319114:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2117 00000100:00100000:3.0:1767320250.319119:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983022336:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320250.319130:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320250.319382:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983021952:12345-0@lo:400:kworker.0 Request processed in 9464us (10108us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320250.319393:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1601 00000100:00100000:1.0:1767320250.319404:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983022208:12345-0@lo:400:kworker.0 Request processed in 9451us (10060us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320250.319427:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1602 00000100:00080000:2.0:1767320250.319459:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00080000:1.0:1767320250.319466:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910380 x1853163983022208/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320266 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320250.319467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320250.319475:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983022336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320250.319487:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983022208:0@lo:400:kworker.0 00000100:00100000:2.0:1767320250.320109:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983022464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320250.320171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320250.320221:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983022848:0@lo:38:kworker.0 00000100:00100000:1.0:1767320250.320241:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022464 00000100:00100000:1.0:1767320250.320248:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022464 00000100:00100000:2.0:1767320250.320253:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320250.320310:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 554 00000100:00100000:1.0:1767320250.320318:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983022464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320250.320333:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320250.320368:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983022464:12345-0@lo:38:kworker.0 Request processed in 51us (196us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320250.321627:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 554 00000100:00100000:1.0:1767320250.321679:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022848 00000100:00100000:1.0:1767320250.321686:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022848 00000100:00100000:1.0:1767320250.321719:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118 00000100:00100000:1.0:1767320250.321727:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983022848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320250.321757:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320250.321828:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983022848:12345-0@lo:38:kworker.0 Request processed in 100us (1577us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320250.321835:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118 00000100:00100000:3.0:1767320250.321888:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983022336:12345-0@lo:38:kworker.0 Request processed in 2764us (2834us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320250.321898:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2117 00000100:00080000:3.0:1767320250.321966:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320250.321971:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320250.321975:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983022848:0@lo:38:kworker.0 00000100:00080000:3.0:1767320250.322019:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320250.322024:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320250.322026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983022464:0@lo:38:kworker.0 00000100:00100000:0.0:1767320250.323422:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983022720:0@lo:400:kworker.0 00010000:00080000:1.0:1767320250.325049:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911880 x1853163983021952/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320266 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320250.325070:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983021952:0@lo:400:kworker.0 00010000:00010000:1.0:1767320250.500553:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320251.526506:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320252.301537:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320252.301853:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670912 00000100:00100000:2.0:1767320252.301865:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670912 00000100:00100000:2.0:1767320252.302038:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1603 00000100:00100000:2.0:1767320252.302045:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915670912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320252.302129:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915670912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (603us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320252.302137:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1603 00000100:00100000:1.0:1767320252.304002:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:0.0:1767320252.304076:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320252.304084:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915670784 00000100:00100000:3.0:1767320252.304093:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915670784 00000100:00100000:3.0:1767320252.304135:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2119 00000100:00100000:3.0:1767320252.304141:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320252.304168:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320252.304223:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915670784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 81us (232us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320252.304230:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2119 00010000:00010000:2.0:1767320252.548561:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320252.869622:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320252.869647:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983022976:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320252.869740:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320252.869872:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983022976 00000100:00100000:1.0:1767320252.869882:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983022976 00000100:00100000:1.0:1767320252.869950:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076 00000100:00100000:1.0:1767320252.869961:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983022976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320252.869992:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320252.870042:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983022976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320252.870050:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076 00000100:00100000:1.0:1767320252.870110:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983022976:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320252.870295:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983023104:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320252.870322:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320252.870476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320252.870708:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023104 00000100:00100000:2.0:1767320252.870715:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023104 00000100:00100000:2.0:1767320252.870983:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077 00000100:00100000:2.0:1767320252.870992:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983023104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320252.871014:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320252.871064:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983023104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (739us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320252.871072:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077 00000100:00100000:0.0:1767320252.871257:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983023104:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320252.871793:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320253.575122:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320254.596522:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320255.428408:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320255.428418:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320255.428480:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320255.428484:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320255.428488:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320255.428495:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5498 00000100:00080000:3.0:1767320255.428500:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320255.428507:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320255.428529:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163983023360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320255.428579:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320255.428582:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320255.428583:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320255.428587:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5498 00000100:00080000:3.0:1767320255.428590:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320255.428593:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320255.428604:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163983023616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320255.428613:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320255.428616:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320255.428618:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320255.428621:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5498 00000100:00080000:3.0:1767320255.428624:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320255.428627:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320255.428682:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163983023744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320255.428704:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320255.428707:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320255.428709:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320255.428712:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5498 00000100:00080000:3.0:1767320255.428715:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320255.428718:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320255.428728:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0380 x1853163983024128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320255.428760:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983023488:0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.428939:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983023232:0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.428974:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320255.429035:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320255.429041:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983024000:0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.429066:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320255.429117:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023232 00000100:00100000:2.0:1767320255.429125:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023232 00000100:00100000:2.0:1767320255.429181:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1604 00000100:00100000:2.0:1767320255.429191:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983023232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.429238:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983023232:12345-0@lo:400:kworker.0 Request processed in 50us (264us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320255.429245:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1604 00000100:00100000:2.0:1767320255.429268:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024000 00000100:00100000:2.0:1767320255.429295:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024000 00000100:00100000:2.0:1767320255.429321:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 997 00000100:00100000:2.0:1767320255.429327:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983024000:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.429357:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983024000:12345-0@lo:400:kworker.0 Request processed in 30us (289us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320255.429363:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 997 00000100:00080000:2.0:1767320255.429378:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320255.429383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320255.429413:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983024000:0@lo:400:kworker.0 00010000:00080000:2.0:1767320255.429424:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34000 x1853163983023232/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320271 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320255.429440:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983023232:0@lo:400:kworker.0 00000100:00100000:3.0:1767320255.429524:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320255.429554:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320255.429560:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983023872:0@lo:400:kworker.0 00000100:00100000:3.0:1767320255.429584:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320255.429653:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023488 00000100:00100000:2.0:1767320255.429657:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023488 00000100:00100000:2.0:1767320255.429697:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1605 00000100:00100000:2.0:1767320255.429704:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983023488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.429739:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983023488:12345-0@lo:400:kworker.0 Request processed in 36us (215us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320255.429745:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1605 00000100:00100000:2.0:1767320255.429760:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023872 00000100:00100000:2.0:1767320255.429763:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023872 00000100:00100000:2.0:1767320255.429783:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998 00000100:00100000:2.0:1767320255.429789:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983023872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320255.429829:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983023872:12345-0@lo:400:kworker.0 Request processed in 26us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320255.429835:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998 00000100:00080000:3.0:1767320255.429981:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320255.429985:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320255.430004:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983023872:0@lo:400:kworker.0 00010000:00080000:3.0:1767320255.430014:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35880 x1853163983023488/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320271 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320255.430023:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983023488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320255.430195:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983023360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320255.430228:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320255.430265:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983023616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320255.430310:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320255.430340:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983023744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320255.430380:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320255.430406:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983024128:0@lo:38:kworker.0 00000100:00100000:1.0:1767320255.430427:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320255.430456:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023744 00000100:00100000:1.0:1767320255.430461:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023744 00000100:00100000:1.0:1767320255.430485:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 555 00000100:00100000:1.0:1767320255.430490:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983023744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320255.430508:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320255.430536:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983023744:12345-0@lo:38:kworker.0 Request processed in 45us (156us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320255.430540:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 555 00000100:00080000:1.0:1767320255.430562:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320255.430565:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320255.430567:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983023744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320255.430733:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023360 00000100:00100000:3.0:1767320255.430737:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023360 00000100:00100000:2.0:1767320255.430745:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983023616 00000100:00100000:2.0:1767320255.430749:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983023616 00000100:00100000:3.0:1767320255.430762:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024128 00000100:00100000:3.0:1767320255.430765:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024128 00000100:00100000:2.0:1767320255.430768:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2120 00000100:00100000:2.0:1767320255.430772:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983023360:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320255.430774:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121 00000100:00100000:3.0:1767320255.430779:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983023616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320255.430782:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320255.430786:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320255.430816:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983023360:12345-0@lo:38:kworker.0 Request processed in 37us (582us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320255.430822:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2120 00000100:00100000:2.0:1767320255.430835:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2122 00000100:00100000:2.0:1767320255.430839:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983024128:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320255.430848:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320255.430976:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983023616:12345-0@lo:38:kworker.0 Request processed in 195us (664us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320255.430983:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121 00000100:00100000:2.0:1767320255.431013:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983024128:12345-0@lo:38:kworker.0 Request processed in 172us (585us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320255.431019:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2122 00000100:00080000:1.0:1767320255.431020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320255.431023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320255.431026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983023360:0@lo:38:kworker.0 00000100:00080000:1.0:1767320255.431035:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320255.431039:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320255.431041:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983023616:0@lo:38:kworker.0 00000100:00080000:1.0:1767320255.431049:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320255.431052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320255.431054:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983024128:0@lo:38:kworker.0 00010000:00010000:2.0:1767320255.620629:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320256.648726:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320257.421884:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320257.422015:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320257.423139:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671168 00000100:00100000:2.0:1767320257.423151:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671168 00000100:00100000:3.0:1767320257.423243:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671040 00000100:00100000:3.0:1767320257.423249:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671040 00000100:00100000:3.0:1767320257.423421:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2123 00000100:00100000:3.0:1767320257.423429:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915671040:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320257.423454:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320257.423531:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915671040:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 99us (1514us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320257.423539:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2123 00000100:00080000:1.0:1767320257.425397:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320257.425535:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1606 00000100:00100000:2.0:1767320257.425550:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915671168:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320257.425727:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915671168:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 180us (3947us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320257.425735:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1606 00010000:00010000:2.0:1767320257.671144:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320257.988725:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983024256:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320257.988819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320257.988898:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320257.988910:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983024384:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320257.988948:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320257.988968:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024256 00000100:00100000:1.0:1767320257.988977:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024256 00000100:00080000:0.0:1767320257.989057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320257.989057:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024384 00000100:00100000:1.0:1767320257.989063:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024384 00000100:00080000:0.0:1767320257.989090:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320257.989096:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1078 00000100:00100000:1.0:1767320257.989110:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983024256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320257.989146:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320257.989208:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983024256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (388us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320257.989213:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079 00000100:00100000:1.0:1767320257.989223:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1078 00000100:00100000:3.0:1767320257.989228:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983024384:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320257.989255:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320257.989316:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983024384:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (368us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320257.989323:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079 00000100:00100000:0.0:1767320257.989373:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983024256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320257.990607:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983024384:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320258.693799:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320259.716530:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320260.548640:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320260.548656:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320260.548716:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320260.548721:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320260.548726:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320260.548731:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983024512:0@lo:400:kworker.0 00000100:00080000:3.0:1767320260.548734:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5503 00000100:00080000:3.0:1767320260.548740:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320260.548747:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320260.548767:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2680 x1853163983024640/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320260.548800:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320260.548804:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320260.548807:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320260.548812:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320260.548813:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5503 00000100:00080000:3.0:1767320260.548817:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320260.548819:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320260.548827:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0a80 x1853163983024896/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320260.548833:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320260.548835:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320260.548836:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320260.548838:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5503 00000100:00080000:3.0:1767320260.548839:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320260.548842:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320260.548848:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3b80 x1853163983025024/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320260.548862:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320260.548865:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320260.548866:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320260.548868:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5503 00000100:00080000:3.0:1767320260.548869:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320260.548871:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320260.548878:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1880 x1853163983025408/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320260.548905:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320260.548915:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983025280:0@lo:400:kworker.0 00000100:00100000:3.0:1767320260.548919:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024512 00000100:00100000:3.0:1767320260.548927:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024512 00000100:00100000:1.0:1767320260.548948:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.549161:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025280 00000100:00100000:0.0:1767320260.549167:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025280 00000100:00100000:0.0:1767320260.549216:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999 00000100:00100000:0.0:1767320260.549227:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983025280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320260.549236:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1607 00000100:00100000:3.0:1767320260.549244:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983024512:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320260.549390:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983024768:0@lo:400:kworker.0 00010000:00080000:1.0:1767320260.549464:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1c00 x1853163983024512/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320276 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320260.549478:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983024512:0@lo:400:kworker.0 00000100:00080000:1.0:1767320260.549506:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320260.549511:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320260.549520:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983025152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320260.552818:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983025280:12345-0@lo:400:kworker.0 Request processed in 3585us (3861us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320260.552846:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999 00000100:00100000:0.0:1767320260.552921:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983025280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320260.552977:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983024640:0@lo:38:kworker.0 00000100:00100000:0.0:1767320260.553046:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.553078:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983024896:0@lo:38:kworker.0 00000100:00100000:0.0:1767320260.553100:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.553118:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983025024:0@lo:38:kworker.0 00000100:00100000:0.0:1767320260.553144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.553171:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983025408:0@lo:38:kworker.0 00000100:00100000:0.0:1767320260.553194:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.553227:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024640 00000100:00100000:0.0:1767320260.553233:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024640 00000100:00100000:0.0:1767320260.554023:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024896 00000100:00100000:0.0:1767320260.554030:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024896 00000100:00100000:0.0:1767320260.554039:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025408 00000100:00100000:0.0:1767320260.554041:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025408 00000100:00100000:0.0:1767320260.554053:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124 00000100:00100000:0.0:1767320260.554060:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983024640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320260.554083:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320260.554131:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983024640:12345-0@lo:38:kworker.0 Request processed in 70us (1084us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320260.554138:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124 00000100:00100000:0.0:1767320260.554152:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2125 00000100:00100000:0.0:1767320260.554156:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983024896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320260.554166:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320260.554189:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983024896:12345-0@lo:38:kworker.0 Request processed in 32us (1088us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320260.554195:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2125 00000100:00100000:0.0:1767320260.554207:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2126 00000100:00100000:0.0:1767320260.554211:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983025408:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320260.554220:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320260.554694:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983025408:12345-0@lo:38:kworker.0 Request processed in 478us (1497us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320260.554701:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2126 00000100:00080000:0.0:1767320260.554747:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320260.554752:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320260.554756:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983024640:0@lo:38:kworker.0 00000100:00080000:0.0:1767320260.554766:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320260.554769:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320260.554772:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983024896:0@lo:38:kworker.0 00000100:00080000:0.0:1767320260.554780:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320260.554783:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320260.554786:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983025408:0@lo:38:kworker.0 00000100:00100000:0.0:1767320260.554953:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025024 00000100:00100000:0.0:1767320260.554960:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025024 00000100:00100000:0.0:1767320260.554985:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 556 00000100:00100000:0.0:1767320260.554990:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983025024:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320260.555001:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320260.555048:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983025024:12345-0@lo:38:kworker.0 Request processed in 56us (1902us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320260.555054:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 556 00000100:00080000:0.0:1767320260.555077:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320260.555081:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320260.555084:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983025024:0@lo:38:kworker.0 00000100:00100000:3.0:1767320260.557200:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983024512:12345-0@lo:400:kworker.0 Request processed in 7941us (8377us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320260.557216:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1607 00000100:00100000:1.0:1767320260.566289:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320260.566361:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025152 00000100:00100000:0.0:1767320260.566372:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025152 00000100:00100000:0.0:1767320260.566463:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000 00000100:00100000:0.0:1767320260.566474:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983025152:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320260.566634:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320260.566822:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983025152:12345-0@lo:400:kworker.0 Request processed in 349us (552us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320260.566832:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000 00000100:00100000:1.0:1767320260.566868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983025152:0@lo:400:kworker.0 00000100:00100000:2.0:1767320260.566993:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320260.567478:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983024768 00000100:00100000:3.0:1767320260.567485:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983024768 00000100:00100000:3.0:1767320260.567557:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1608 00000100:00100000:3.0:1767320260.567565:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983024768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320260.567626:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983024768:12345-0@lo:400:kworker.0 Request processed in 62us (633us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320260.567632:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1608 00010000:00080000:2.0:1767320260.567747:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0e00 x1853163983024768/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320276 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320260.567759:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983024768:0@lo:400:kworker.0 00000100:00080000:1.0:1767320260.567821:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320260.740577:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320261.764880:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320262.539758:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320262.539850:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320262.539995:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671296 00000100:00100000:0.0:1767320262.540004:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671296 00000100:00100000:0.0:1767320262.540055:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671424 00000100:00100000:0.0:1767320262.540059:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671424 00000100:00100000:0.0:1767320262.540115:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1609 00000100:00100000:0.0:1767320262.540128:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915671424:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767320262.540144:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320262.540214:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915671424:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320262.540222:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1609 00000100:00100000:0.0:1767320262.540242:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2127 00000100:00100000:0.0:1767320262.540245:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915671296:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320262.540263:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320262.540321:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915671296:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 74us (574us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320262.540326:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2127 00000400:00000100:1.0:1767320262.543599:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915671552 offset 0 length 520: 4 00010000:00010000:1.0:1767320262.791906:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320263.110713:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320263.110737:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983025664:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320263.110777:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320263.110790:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983025536:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320263.110812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320263.110850:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320263.110911:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025664 00000100:00100000:1.0:1767320263.110915:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025536 00000100:00100000:0.0:1767320263.110919:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025664 00000100:00100000:1.0:1767320263.110921:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025536 00000100:00100000:0.0:1767320263.111034:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080 00000100:00100000:0.0:1767320263.111042:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983025664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320263.111072:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320263.111114:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983025664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (303us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320263.111121:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080 00000100:00100000:0.0:1767320263.111164:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983025664:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320263.113524:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1081 00000100:00100000:1.0:1767320263.113540:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983025536:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320263.113572:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320263.113639:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983025536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (2788us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320263.113646:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1081 00000100:00100000:3.0:1767320263.113719:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983025536:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320263.118789:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320263.118799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320263.815769:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320264.836514:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320265.668396:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320240/real 1767320240] req@ffff934609a34380 x1853163983019264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320265 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320265.668429:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609a34380 x1853163983019264/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320265 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320265.668432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668440:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320265.668447:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320265.668481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320265.668487:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983019264:0@lo:250:kworker.0 00000100:00080000:2.0:1767320265.668510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320265.668518:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320265.668523:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5509 00000100:00080000:2.0:1767320265.668528:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320265.668530:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983025792:0@lo:400:kworker.0 00000100:00080000:2.0:1767320265.668537:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320265.668562:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358e00 x1853163983025920/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320265.668601:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320265.668602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320265.668610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320265.668614:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5509 00000100:00080000:2.0:1767320265.668617:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320265.668621:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320265.668621:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320265.668629:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983026048:0@lo:400:kworker.0 00000100:00100000:2.0:1767320265.668632:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163983026176/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320265.668641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320265.668646:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320265.668650:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5509 00000100:00080000:2.0:1767320265.668653:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320265.668657:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320265.668664:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320265.668667:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359180 x1853163983026304/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320265.668689:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983026432:0@lo:400:kworker.0 00000100:00100000:0.0:1767320265.668694:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983025920:0@lo:38:kworker.0 00000100:00080000:2.0:1767320265.668695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320265.668705:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320265.668709:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5509 00000100:00080000:2.0:1767320265.668712:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320265.668715:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320265.668718:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320265.668727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435a300 x1853163983026688/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320265.668738:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320265.668744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320265.668748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320265.668750:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320265.668753:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5488 00000100:00100000:1.0:1767320265.668754:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026432 00000100:00080000:2.0:1767320265.668756:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320265.668760:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320265.668761:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026432 00000100:00100000:2.0:1767320265.668772:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b480 x1853163983026816/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320265.668775:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983026176:0@lo:38:kworker.0 00000100:00100000:0.0:1767320265.668809:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320265.668815:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983026560:0@lo:400:kworker.0 00000100:00100000:1.0:1767320265.668816:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001 00000100:00100000:1.0:1767320265.668826:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983026432:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320265.668835:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983026304:0@lo:38:kworker.0 00000100:00100000:2.0:1767320265.668847:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320265.668875:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320265.668882:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983026432:12345-0@lo:400:kworker.0 Request processed in 58us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320265.668883:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026560 00000100:00100000:2.0:1767320265.668887:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026560 00000100:00100000:1.0:1767320265.668891:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001 00000100:00100000:1.0:1767320265.668912:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1002 00000100:00100000:0.0:1767320265.668919:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983026688:0@lo:38:kworker.0 00000100:00100000:1.0:1767320265.668921:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983026560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320265.668960:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320265.668968:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983026560:12345-0@lo:400:kworker.0 Request processed in 48us (120us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320265.668977:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025792 00000100:00100000:0.0:1767320265.668984:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983026816:0@lo:250:kworker.0 00000100:00100000:3.0:1767320265.668985:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025792 00000400:00000100:0.0:1767320265.669010:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983026816 offset 0 length 520: 4 00000100:00100000:3.0:1767320265.669018:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026048 00000100:00100000:3.0:1767320265.669022:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026048 00000100:00100000:3.0:1767320265.669042:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983025920 00000100:00100000:3.0:1767320265.669045:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983025920 00000100:00100000:0.0:1767320265.669051:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026176 00000100:00100000:3.0:1767320265.669056:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026688 00000100:00100000:0.0:1767320265.669057:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026176 00000100:00100000:3.0:1767320265.669061:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026688 00000100:00100000:3.0:1767320265.669080:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1610 00000100:00100000:3.0:1767320265.669086:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983025792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320265.669137:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983025792:12345-0@lo:400:kworker.0 Request processed in 36us (523us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320265.669143:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1610 00000100:00100000:3.0:1767320265.669156:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1611 00000100:00100000:3.0:1767320265.669162:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983026048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320265.669187:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2128 00000100:00100000:3.0:1767320265.669189:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983026048:12345-0@lo:400:kworker.0 Request processed in 25us (523us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320265.669200:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983025920:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320265.669200:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1611 00000100:00100000:3.0:1767320265.669215:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2129 00010000:00020000:0.0:1767320265.669222:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320265.669222:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983026176:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320265.669236:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320265.669289:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983026176:12345-0@lo:38:kworker.0 Request processed in 48us (463us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320265.669296:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2129 00000100:00100000:3.0:1767320265.669310:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130 00000100:00100000:0.0:1767320265.669314:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983025920:12345-0@lo:38:kworker.0 Request processed in 70us (534us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320265.669315:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983026688:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320265.669328:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2128 00010000:00020000:3.0:1767320265.669330:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320265.669354:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983026688:12345-0@lo:38:kworker.0 Request processed in 39us (395us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320265.669359:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026304 00000100:00100000:3.0:1767320265.669362:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130 00000100:00100000:0.0:1767320265.669365:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026304 00000100:00100000:0.0:1767320265.669392:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 557 00000100:00100000:0.0:1767320265.669397:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983026304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320265.669405:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:3.0:1767320265.669418:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435a680 x1853163983025792/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320281 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320265.669429:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983026304:12345-0@lo:38:kworker.0 Request processed in 31us (555us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320265.669433:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983025792:0@lo:400:kworker.0 00000100:00100000:0.0:1767320265.669437:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 557 00000100:00080000:0.0:1767320265.669477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320265.669480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320265.669482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983026688:0@lo:38:kworker.0 00000100:00080000:0.0:1767320265.669487:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320265.669489:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320265.669491:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983025920:0@lo:38:kworker.0 00000100:00080000:0.0:1767320265.669496:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320265.669497:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320265.669499:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983026176:0@lo:38:kworker.0 00000100:00080000:0.0:1767320265.669503:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320265.669506:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320265.669508:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983026304:0@lo:38:kworker.0 00000100:00080000:3.0:1767320265.669521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320265.669528:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320265.669530:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320265.669545:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983026432:0@lo:400:kworker.0 00010000:00080000:0.0:1767320265.669556:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358a80 x1853163983026048/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320281 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320265.669562:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983026048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320265.669570:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983026560:0@lo:400:kworker.0 00000100:00100000:1.0:1767320265.674750:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1002 00010000:00010000:1.0:1767320265.862507:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320266.885194:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320267.667147:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320267.667199:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320267.669678:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671808 00000100:00100000:3.0:1767320267.669693:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671808 00000100:00100000:3.0:1767320267.669798:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671680 00000100:00100000:3.0:1767320267.669801:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671680 00000100:00100000:3.0:1767320267.669832:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1612 00000100:00100000:3.0:1767320267.669843:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915671808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320267.669928:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915671808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (2791us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320267.669935:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1612 00000100:00100000:3.0:1767320267.669954:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2131 00000100:00100000:3.0:1767320267.669958:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915671680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320267.669978:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320267.670006:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915671680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (2807us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320267.670013:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2131 00000100:00080000:2.0:1767320267.670085:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320267.908726:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320268.228724:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983026944:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320268.228796:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320268.229062:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983026944 00000100:00100000:1.0:1767320268.229071:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983026944 00000100:00100000:1.0:1767320268.229135:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082 00000100:00100000:1.0:1767320268.229145:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983026944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320268.229175:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320268.229222:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983026944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320268.229229:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082 00000100:00080000:3.0:1767320268.229290:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320268.229327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983026944:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320268.232667:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983027072:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320268.232735:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320268.232836:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027072 00000100:00100000:1.0:1767320268.232846:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027072 00000100:00100000:1.0:1767320268.233398:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083 00000100:00100000:1.0:1767320268.233420:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983027072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320268.233451:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320268.233514:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983027072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (779us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320268.233522:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083 00000100:00100000:0.0:1767320268.233585:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983027072:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320268.233857:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320268.932753:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320269.956492:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320270.788489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320270.788500:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320270.790024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320270.790035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320270.790039:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320270.790048:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5514 00000100:00080000:2.0:1767320270.790060:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320270.790068:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320270.790115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359880 x1853163983027328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320270.790156:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983027200:0@lo:400:kworker.0 00000100:00080000:2.0:1767320270.790172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320270.790175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320270.790177:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320270.790181:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5514 00000100:00080000:2.0:1767320270.790183:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320270.790187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320270.790197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435ad80 x1853163983027584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320270.790205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320270.790208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320270.790209:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320270.790212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5514 00000100:00080000:2.0:1767320270.790214:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320270.790217:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320270.790226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359180 x1853163983027712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320270.790227:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320270.790246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320270.790249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320270.790251:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320270.790254:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5514 00000100:00080000:2.0:1767320270.790256:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320270.790259:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320270.790265:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320270.790267:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435a300 x1853163983028096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320270.790311:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983027456:0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.790332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983027328:0@lo:38:kworker.0 00000100:00100000:2.0:1767320270.790340:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320270.790371:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320270.790372:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320270.790376:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983027840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.790394:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983027584:0@lo:38:kworker.0 00000100:00100000:2.0:1767320270.790402:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320270.790412:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320270.790427:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983027712:0@lo:38:kworker.0 00000100:00100000:3.0:1767320270.790448:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320270.790452:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027200 00000100:00100000:2.0:1767320270.790460:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027200 00000100:00100000:3.0:1767320270.790464:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983028096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320270.790482:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320270.790506:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027456 00000100:00100000:3.0:1767320270.790506:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027328 00000100:00100000:2.0:1767320270.790509:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027456 00000100:00100000:3.0:1767320270.790509:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027328 00000100:00100000:2.0:1767320270.790522:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028096 00000100:00100000:3.0:1767320270.790522:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027584 00000100:00100000:2.0:1767320270.790524:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028096 00000100:00100000:3.0:1767320270.790524:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027584 00000100:00100000:2.0:1767320270.790535:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1613 00000100:00100000:2.0:1767320270.790543:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983027200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320270.790591:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983027200:12345-0@lo:400:kworker.0 Request processed in 49us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320270.790615:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983027968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320270.790643:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767320270.790676:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358700 x1853163983027200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320286 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320270.790688:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983027200:0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.791841:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1614 00000100:00100000:3.0:1767320270.791872:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983027456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.791956:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983027456:12345-0@lo:400:kworker.0 Request processed in 105us (1614us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320270.791966:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1614 00000100:00100000:3.0:1767320270.791994:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2132 00000100:00100000:3.0:1767320270.791998:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983027328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320270.792018:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320270.792066:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983027328:12345-0@lo:38:kworker.0 Request processed in 67us (1694us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320270.792073:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2132 00000100:00100000:3.0:1767320270.792088:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133 00000100:00100000:3.0:1767320270.792092:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983027584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320270.792102:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320270.792126:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983027584:12345-0@lo:38:kworker.0 Request processed in 33us (1713us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320270.792133:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133 00000100:00100000:3.0:1767320270.792145:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2134 00000100:00100000:3.0:1767320270.792149:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983028096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320270.792158:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320270.792181:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983028096:12345-0@lo:38:kworker.0 Request processed in 32us (1699us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320270.792187:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2134 00000100:00100000:3.0:1767320270.792210:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027712 00000100:00100000:3.0:1767320270.792216:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027712 00000100:00100000:3.0:1767320270.792241:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 558 00000100:00100000:3.0:1767320270.792245:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983027712:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320270.792255:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320270.792450:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983027712:12345-0@lo:38:kworker.0 Request processed in 201us (1999us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320270.792457:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 558 00000100:00080000:3.0:1767320270.792475:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320270.792512:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358a80 x1853163983027456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320286 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320270.792524:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983027456:0@lo:400:kworker.0 00000100:00080000:3.0:1767320270.792556:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320270.792561:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320270.792565:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983027328:0@lo:38:kworker.0 00000100:00080000:3.0:1767320270.792574:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320270.792578:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320270.792581:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983027584:0@lo:38:kworker.0 00000100:00080000:3.0:1767320270.792591:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320270.792594:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320270.792597:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983027712:0@lo:38:kworker.0 00000100:00080000:3.0:1767320270.792605:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320270.792608:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320270.792611:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983028096:0@lo:38:kworker.0 00000100:00100000:3.0:1767320270.792661:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027840 00000100:00100000:3.0:1767320270.792665:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027840 00000100:00100000:3.0:1767320270.792700:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983027968 00000100:00100000:3.0:1767320270.792703:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983027968 00000100:00100000:3.0:1767320270.792808:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1003 00000100:00100000:3.0:1767320270.792814:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983027840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.792854:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983027840:12345-0@lo:400:kworker.0 Request processed in 40us (2450us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320270.792860:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1003 00000100:00100000:3.0:1767320270.792872:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004 00000100:00100000:3.0:1767320270.792878:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983027968:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.792912:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983027968:12345-0@lo:400:kworker.0 Request processed in 34us (2268us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320270.792918:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004 00000100:00080000:3.0:1767320270.792937:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320270.792939:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320270.792954:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983027840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320270.792999:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983027968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320270.794250:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1613 00010000:00010000:2.0:1767320270.981913:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320272.004521:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320272.780096:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320272.780190:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320272.782073:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672064 00000100:00100000:2.0:1767320272.782085:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672064 00000100:00100000:2.0:1767320272.782155:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915671936 00000100:00100000:2.0:1767320272.782159:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915671936 00000100:00100000:2.0:1767320272.782190:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1615 00000100:00100000:2.0:1767320272.782199:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915672064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320272.782203:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2135 00000100:00100000:1.0:1767320272.782209:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915671936:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320272.782230:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320272.782312:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320272.782423:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915672064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 223us (2336us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320272.782431:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1615 00000100:00100000:1.0:1767320272.783367:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915671936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 1150us (3170us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320272.783378:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2135 00010000:00010000:2.0:1767320273.028464:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320273.348591:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983028224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320273.348619:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320273.348639:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983028352:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320273.348680:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320273.348794:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028352 00000100:00100000:2.0:1767320273.348803:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028352 00000100:00100000:2.0:1767320273.348875:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084 00000100:00100000:2.0:1767320273.348885:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983028352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320273.348917:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320273.348963:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983028352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (283us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320273.348971:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084 00000100:00080000:1.0:1767320273.349226:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320273.349267:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983028352:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320273.349555:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320273.349606:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028224 00000100:00100000:1.0:1767320273.349612:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028224 00000100:00100000:1.0:1767320273.349663:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085 00000100:00100000:1.0:1767320273.349671:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983028224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320273.349690:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320273.349736:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983028224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (182us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320273.349743:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085 00000100:00080000:0.0:1767320273.349835:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320273.350354:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983028224:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320274.052520:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320275.076803:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320275.908470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320275.908477:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320275.908538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320275.908542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320275.908544:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320275.908551:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5519 00000100:00080000:2.0:1767320275.908555:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320275.908562:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320275.908584:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35180 x1853163983028608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320275.908623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320275.908627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320275.908629:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320275.908634:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5519 00000100:00080000:2.0:1767320275.908636:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320275.908640:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320275.908651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163983028864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320275.908657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320275.908659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320275.908660:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320275.908662:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5519 00000100:00080000:2.0:1767320275.908664:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320275.908666:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320275.908675:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52680 x1853163983028992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320275.908711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320275.908714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320275.908715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320275.908717:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5519 00000100:00080000:2.0:1767320275.908718:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320275.908721:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320275.908727:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d500 x1853163983029376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320275.908808:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983028480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320275.908870:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320275.908872:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983029248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320275.908917:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320275.908944:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983029120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320275.908967:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320275.908984:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029248 00000100:00100000:1.0:1767320275.908988:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320275.908993:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983028736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320275.908993:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029248 00000100:00100000:1.0:1767320275.909011:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320275.909035:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028480 00000100:00100000:1.0:1767320275.909038:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028480 00000100:00100000:3.0:1767320275.909055:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029120 00000100:00100000:3.0:1767320275.909060:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029120 00000100:00100000:1.0:1767320275.909065:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028736 00000100:00100000:1.0:1767320275.909069:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028736 00000100:00100000:3.0:1767320275.909092:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005 00000100:00100000:1.0:1767320275.909096:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1616 00000100:00100000:1.0:1767320275.909108:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983028480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320275.909109:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983029248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320275.909170:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983029248:12345-0@lo:400:kworker.0 Request processed in 67us (252us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320275.909171:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983028480:12345-0@lo:400:kworker.0 Request processed in 67us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320275.909185:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1616 00000100:00100000:3.0:1767320275.909189:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005 00000100:00100000:1.0:1767320275.909203:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1617 00000100:00100000:3.0:1767320275.909214:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1006 00000100:00100000:1.0:1767320275.909215:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983028736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320275.909226:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983029120:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320275.909524:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320275.909528:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320275.909529:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320275.909562:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983029248:0@lo:400:kworker.0 00010000:00080000:2.0:1767320275.909609:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36d80 x1853163983028480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320291 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320275.909619:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983028480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320275.909659:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983028736:12345-0@lo:400:kworker.0 Request processed in 89us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320275.909666:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1617 00010000:00080000:1.0:1767320275.909688:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35880 x1853163983028736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320291 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320275.909694:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983028736:0@lo:400:kworker.0 00000100:00100000:0.0:1767320275.909838:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983028608:0@lo:38:kworker.0 00000100:00100000:0.0:1767320275.909868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320275.909878:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983029120:12345-0@lo:400:kworker.0 Request processed in 654us (908us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320275.909887:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1006 00000100:00100000:0.0:1767320275.909895:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983028864:0@lo:38:kworker.0 00000100:00100000:0.0:1767320275.909912:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320275.909924:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983028992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320275.909948:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320275.909971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983029376:0@lo:38:kworker.0 00000100:00100000:0.0:1767320275.909984:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320275.910002:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028608 00000100:00100000:0.0:1767320275.910005:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028608 00000100:00100000:0.0:1767320275.910021:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028864 00000100:00100000:0.0:1767320275.910022:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028864 00000100:00100000:0.0:1767320275.910029:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029376 00000100:00100000:0.0:1767320275.910030:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029376 00000100:00100000:0.0:1767320275.910037:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136 00000100:00100000:0.0:1767320275.910040:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983028608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320275.910056:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320275.910079:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983028608:12345-0@lo:38:kworker.0 Request processed in 38us (211us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320275.910083:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136 00000100:00100000:0.0:1767320275.910090:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2137 00000100:00100000:0.0:1767320275.910092:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983028864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320275.910098:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320275.910110:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983028864:12345-0@lo:38:kworker.0 Request processed in 16us (198us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320275.910113:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2137 00000100:00100000:0.0:1767320275.910394:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2138 00000100:00100000:0.0:1767320275.910398:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983029376:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320275.910405:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320275.910425:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983029376:12345-0@lo:38:kworker.0 Request processed in 26us (440us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320275.910428:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2138 00000100:00080000:0.0:1767320275.910448:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320275.910451:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320275.910453:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983028608:0@lo:38:kworker.0 00000100:00080000:0.0:1767320275.910458:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320275.910460:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320275.910462:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983028864:0@lo:38:kworker.0 00000100:00080000:0.0:1767320275.910466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320275.910468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320275.910469:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983029376:0@lo:38:kworker.0 00000100:00100000:0.0:1767320275.910497:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983028992 00000100:00100000:0.0:1767320275.910500:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983028992 00000100:00100000:0.0:1767320275.910513:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 559 00000100:00100000:0.0:1767320275.910515:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983028992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320275.910522:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320275.910540:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983028992:12345-0@lo:38:kworker.0 Request processed in 23us (591us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320275.910543:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 559 00000100:00080000:0.0:1767320275.910556:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320275.910559:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320275.910562:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983028992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320275.914843:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983029120:0@lo:400:kworker.0 00010000:00010000:2.0:1767320276.100527:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320277.127954:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320277.900571:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320277.901352:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672192 00000100:00100000:2.0:1767320277.901364:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672192 00000100:00100000:2.0:1767320277.901512:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2139 00000100:00100000:2.0:1767320277.901521:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915672192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320277.901543:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320277.901636:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915672192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 114us (1084us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320277.901643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2139 00000100:00100000:0.0:1767320277.903569:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320277.904123:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672320 00000100:00100000:2.0:1767320277.904132:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672320 00000100:00080000:0.0:1767320277.904494:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320277.904619:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1618 00000100:00100000:2.0:1767320277.904636:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915672320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320277.905576:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915672320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (1173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320277.905588:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1618 00010000:00010000:1.0:1767320278.148486:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320278.469056:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320278.469085:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983029504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320278.469178:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320278.469435:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320278.469446:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983029632:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320278.469476:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320278.474798:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029504 00000100:00100000:2.0:1767320278.474820:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029504 00000100:00100000:2.0:1767320278.475026:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029632 00000100:00100000:2.0:1767320278.475031:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029632 00000100:00100000:2.0:1767320278.475052:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086 00000100:00100000:2.0:1767320278.475061:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983029504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320278.475099:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320278.475210:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983029504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (6021us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320278.475265:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086 00000100:00100000:2.0:1767320278.475741:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1087 00000100:00100000:2.0:1767320278.475750:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983029632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320278.475771:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320278.475804:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983029632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (6327us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320278.475811:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1087 00000100:00080000:2.0:1767320278.475862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320278.475867:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320278.477956:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983029504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320278.478256:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983029632:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320279.172498:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320280.196460:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320281.033090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320281.033102:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320281.033159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320281.033163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320281.033166:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320281.033207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5524 00000100:00080000:0.0:1767320281.033219:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320281.033226:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320281.033255:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320281.033290:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983029760:0@lo:400:kworker.0 00000100:00100000:0.0:1767320281.033299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3480 x1853163983029888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320281.033335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320281.033339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320281.033341:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320281.033345:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5524 00000100:00080000:0.0:1767320281.033348:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320281.033353:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320281.033365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0700 x1853163983030144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320281.033375:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320281.033376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320281.033381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320281.033383:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320281.033387:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5524 00000100:00080000:0.0:1767320281.033390:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320281.033393:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320281.033405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2a00 x1853163983030272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320281.033429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320281.033432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320281.033435:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320281.033438:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5524 00000100:00080000:0.0:1767320281.033441:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320281.033445:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320281.033456:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983030528:0@lo:400:kworker.0 00000100:00100000:0.0:1767320281.033459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0380 x1853163983030656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320281.033487:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320281.033583:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030528 00000100:00100000:3.0:1767320281.033591:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030528 00000100:00100000:0.0:1767320281.033633:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029760 00000100:00100000:0.0:1767320281.033638:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029760 00000100:00100000:1.0:1767320281.033713:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983030400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320281.033745:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320281.033769:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320281.033775:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983030016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320281.033797:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320281.033824:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030400 00000100:00100000:1.0:1767320281.033827:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030400 00000100:00100000:3.0:1767320281.043166:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007 00000100:00100000:3.0:1767320281.043195:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983030528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320281.043229:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983029888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320281.043330:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983030528:12345-0@lo:400:kworker.0 Request processed in 143us (9841us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320281.043338:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007 00000100:00100000:3.0:1767320281.043384:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030016 00000100:00100000:3.0:1767320281.043392:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030016 00000100:00100000:2.0:1767320281.043407:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320281.043508:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983030144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320281.043534:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320281.043553:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983030272:0@lo:38:kworker.0 00000100:00100000:2.0:1767320281.043578:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320281.043599:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983030656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320281.043618:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320281.046109:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983029888 00000100:00100000:0.0:1767320281.048625:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983029888 00000100:00100000:0.0:1767320281.048682:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030144 00000100:00100000:0.0:1767320281.048687:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030144 00000100:00100000:0.0:1767320281.048696:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030656 00000100:00100000:0.0:1767320281.048698:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030656 00000100:00100000:0.0:1767320281.048716:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1619 00000100:00100000:0.0:1767320281.048734:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983029760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320281.048826:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983029760:12345-0@lo:400:kworker.0 Request processed in 101us (15452us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320281.048833:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1619 00000100:00100000:0.0:1767320281.048850:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2140 00000100:00100000:0.0:1767320281.048854:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983029888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320281.048875:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320281.048917:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983029888:12345-0@lo:38:kworker.0 Request processed in 61us (5606us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320281.048923:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2140 00000100:00100000:0.0:1767320281.048934:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2141 00000100:00100000:0.0:1767320281.048937:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983030144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320281.048946:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320281.048966:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983030144:12345-0@lo:38:kworker.0 Request processed in 28us (5432us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320281.048972:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2141 00000100:00100000:0.0:1767320281.048981:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142 00000100:00100000:0.0:1767320281.048985:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983030656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320281.048993:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320281.049012:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983030656:12345-0@lo:38:kworker.0 Request processed in 26us (5393us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320281.049017:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142 00000100:00100000:0.0:1767320281.049044:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030272 00000100:00100000:0.0:1767320281.049048:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030272 00000100:00100000:0.0:1767320281.049066:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 560 00000100:00100000:0.0:1767320281.049070:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983030272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320281.049078:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320281.049102:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983030272:12345-0@lo:38:kworker.0 Request processed in 31us (5523us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320281.049108:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 560 00000100:00100000:0.0:1767320281.049224:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983030528:0@lo:400:kworker.0 00010000:00080000:0.0:1767320281.049242:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f52680 x1853163983029760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320297 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320281.049253:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983029760:0@lo:400:kworker.0 00000100:00080000:0.0:1767320281.049337:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320281.049343:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320281.049346:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983029888:0@lo:38:kworker.0 00000100:00080000:0.0:1767320281.050315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320281.050320:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320281.050324:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983030144:0@lo:38:kworker.0 00000100:00080000:0.0:1767320281.050336:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320281.050340:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320281.050342:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983030272:0@lo:38:kworker.0 00000100:00080000:0.0:1767320281.050351:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320281.050354:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320281.050357:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983030656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320281.050835:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008 00000100:00100000:1.0:1767320281.050849:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983030400:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320281.050958:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983030400:12345-0@lo:400:kworker.0 Request processed in 113us (17212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320281.050966:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008 00000100:00100000:1.0:1767320281.051010:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983030400:0@lo:400:kworker.0 00000100:00080000:2.0:1767320281.053303:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320281.053360:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620 00000100:00100000:3.0:1767320281.053374:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983030016:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320281.053468:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983030016:12345-0@lo:400:kworker.0 Request processed in 98us (19669us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320281.053475:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620 00010000:00080000:1.0:1767320281.053562:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1500 x1853163983030016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320297 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320281.053580:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983030016:0@lo:400:kworker.0 00000100:00080000:2.0:1767320281.054212:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320281.054216:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320281.220475:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320282.244697:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320283.023896:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320283.023968:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320283.024002:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672448 00000100:00100000:3.0:1767320283.024012:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672448 00000100:00100000:3.0:1767320283.024200:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672576 00000100:00100000:3.0:1767320283.024205:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672576 00000100:00080000:2.0:1767320283.024294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320283.024362:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1621 00000100:00100000:1.0:1767320283.024374:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915672576:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320283.024450:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2143 00000100:00100000:0.0:1767320283.024457:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915672448:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320283.024478:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320283.024499:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915672576:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 127us (531us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320283.024505:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915672448:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (625us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320283.024506:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1621 00000100:00100000:0.0:1767320283.024511:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2143 00010000:00010000:3.0:1767320283.268621:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320283.589555:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983030912:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320283.589634:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320283.589742:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030912 00000100:00100000:2.0:1767320283.589753:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030912 00000100:00100000:2.0:1767320283.589827:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088 00000100:00100000:2.0:1767320283.589839:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983030912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320283.589870:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320283.589915:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983030912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320283.589922:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088 00000100:00100000:2.0:1767320283.589943:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320283.589952:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983030784:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320283.589976:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320283.590002:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983030784 00000100:00100000:2.0:1767320283.590006:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983030784 00000100:00100000:2.0:1767320283.590031:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089 00000100:00100000:2.0:1767320283.590038:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983030784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320283.590055:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320283.590423:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983030912:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320283.590446:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320283.590450:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320283.591887:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983030784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1842us (1902us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320283.591899:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089 00000100:00100000:3.0:1767320283.591966:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983030784:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320284.292581:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320285.316547:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320286.148465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320286.148482:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320286.148547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320286.148552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320286.148556:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320286.148564:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5529 00000100:00080000:0.0:1767320286.148569:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320286.148575:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320286.148598:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0700 x1853163983031168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320286.148636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320286.148640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320286.148642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320286.148645:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5529 00000100:00080000:0.0:1767320286.148648:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320286.148652:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320286.148675:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1f80 x1853163983031424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320286.148685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320286.148689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320286.148690:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320286.148694:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5529 00000100:00080000:0.0:1767320286.148697:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320286.148700:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320286.148713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0000 x1853163983031552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320286.148738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320286.148741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320286.148743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320286.148747:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5529 00000100:00080000:0.0:1767320286.148750:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320286.148753:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320286.148767:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163983031936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320286.148942:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983031808:0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149010:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320286.149094:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320286.149101:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983031040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149125:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320286.149156:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983031296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149178:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320286.149196:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320286.149201:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983031680:0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149221:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320286.149250:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031040 00000100:00100000:3.0:1767320286.149256:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031040 00000100:00100000:3.0:1767320286.149333:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031296 00000100:00100000:3.0:1767320286.149336:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031296 00000100:00100000:3.0:1767320286.149360:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1622 00000100:00100000:3.0:1767320286.149369:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983031040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149419:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983031040:12345-0@lo:400:kworker.0 Request processed in 51us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320286.149426:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1622 00000100:00100000:3.0:1767320286.149441:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1623 00000100:00100000:3.0:1767320286.149447:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983031296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149475:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983031296:12345-0@lo:400:kworker.0 Request processed in 29us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320286.149481:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1623 00000100:00100000:3.0:1767320286.149500:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031808 00000100:00100000:3.0:1767320286.149503:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031808 00000100:00100000:3.0:1767320286.149527:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031680 00000100:00100000:3.0:1767320286.149530:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031680 00000100:00100000:3.0:1767320286.149547:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1009 00000100:00100000:3.0:1767320286.149552:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983031808:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149582:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983031808:12345-0@lo:400:kworker.0 Request processed in 30us (572us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320286.149588:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1009 00000100:00100000:3.0:1767320286.149598:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1010 00000100:00100000:3.0:1767320286.149605:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983031680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320286.149629:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983031680:12345-0@lo:400:kworker.0 Request processed in 25us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320286.149635:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1010 00000100:00080000:3.0:1767320286.149650:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320286.149654:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320286.149656:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320286.149892:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983031168:0@lo:38:kworker.0 00000100:00100000:2.0:1767320286.149925:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320286.149953:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983031424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320286.149975:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320286.149992:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983031552:0@lo:38:kworker.0 00000100:00100000:2.0:1767320286.150016:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320286.150038:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983031936:0@lo:38:kworker.0 00000100:00100000:2.0:1767320286.150058:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320286.160458:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983031680:0@lo:400:kworker.0 00010000:00080000:1.0:1767320286.160484:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f3480 x1853163983031296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320302 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320286.160495:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983031296:0@lo:400:kworker.0 00010000:00080000:1.0:1767320286.161448:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2a00 x1853163983031040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320302 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320286.161463:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983031040:0@lo:400:kworker.0 00000100:00100000:1.0:1767320286.161479:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983031808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320286.162128:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031168 00000100:00100000:0.0:1767320286.162140:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031168 00000100:00100000:0.0:1767320286.162379:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031424 00000100:00100000:0.0:1767320286.162384:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031424 00000100:00100000:3.0:1767320286.162540:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031936 00000100:00100000:3.0:1767320286.162545:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031936 00000100:00100000:3.0:1767320286.162575:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2144 00000100:00100000:3.0:1767320286.162582:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031168:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320286.162603:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320286.162686:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031168:12345-0@lo:38:kworker.0 Request processed in 103us (12760us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320286.162694:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2144 00000100:00100000:3.0:1767320286.162711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145 00000100:00100000:3.0:1767320286.162715:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320286.162724:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320286.162751:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031424:12345-0@lo:38:kworker.0 Request processed in 34us (12775us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320286.162756:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145 00000100:00100000:3.0:1767320286.162767:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2146 00000100:00100000:3.0:1767320286.162771:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320286.162779:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320286.162802:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983031936:12345-0@lo:38:kworker.0 Request processed in 29us (12742us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320286.162807:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2146 00000100:00100000:3.0:1767320286.162839:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983031552 00000100:00100000:3.0:1767320286.162843:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983031552 00000100:00100000:3.0:1767320286.162863:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 561 00000100:00100000:3.0:1767320286.162868:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983031552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320286.162877:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320286.162902:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983031552:12345-0@lo:38:kworker.0 Request processed in 33us (12886us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320286.162908:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 561 00000100:00080000:3.0:1767320286.162945:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320286.162951:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320286.162955:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983031936:0@lo:38:kworker.0 00000100:00080000:3.0:1767320286.162976:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320286.162980:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320286.162983:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983031168:0@lo:38:kworker.0 00000100:00080000:3.0:1767320286.162991:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320286.162995:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320286.162999:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983031424:0@lo:38:kworker.0 00000100:00080000:3.0:1767320286.163007:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320286.163010:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320286.163014:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983031552:0@lo:38:kworker.0 00010000:00010000:3.0:1767320286.340602:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767320287.120476:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915672704 offset 0 length 520: 4 00000100:00100000:0.0:1767320287.120596:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320287.120703:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320287.120842:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672832 00000100:00100000:1.0:1767320287.120853:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672832 00000100:00100000:1.0:1767320287.120897:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915672960 00000100:00100000:1.0:1767320287.120901:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915672960 00000100:00100000:1.0:1767320287.120959:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1624 00000100:00100000:1.0:1767320287.120969:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915672960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320287.121048:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915672960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320287.121056:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1624 00000100:00100000:1.0:1767320287.121105:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2147 00000100:00100000:1.0:1767320287.121109:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915672832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320287.121131:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320287.121156:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915672832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (562us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320287.121163:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2147 00000100:00080000:3.0:1767320287.121378:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320287.366685:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320288.388533:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320288.708953:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983032064:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320288.709185:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320288.709194:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983032192:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320288.709223:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320288.709530:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032192 00000100:00100000:2.0:1767320288.709538:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032192 00000100:00100000:2.0:1767320288.709594:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1090 00000100:00100000:2.0:1767320288.709601:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983032192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320288.709626:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320288.709659:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983032192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (435us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320288.709664:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1090 00000100:00100000:1.0:1767320288.709734:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320288.709864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320288.710180:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032064 00000100:00100000:0.0:1767320288.710186:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032064 00000100:00100000:0.0:1767320288.710285:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091 00000100:00100000:0.0:1767320288.710292:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983032064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320288.710310:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320288.710353:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983032064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 62us (619us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320288.710357:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091 00000100:00100000:0.0:1767320288.710400:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983032192:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320288.710661:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983032064:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320288.710767:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767320289.413975:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320290.244447:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320265/real 1767320265] req@ffff93460435b480 x1853163983026816/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320290 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320290.244486:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460435b480 x1853163983026816/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320290 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320290.244506:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320290.244535:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320290.244539:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983026816:0@lo:250:kworker.0 00000100:00080000:3.0:1767320290.244632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320290.244637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320290.244639:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320290.244643:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5514 00000100:00080000:3.0:1767320290.244646:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320290.244652:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320290.244690:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ca80 x1853163983032320/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320290.244753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320290.244756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320290.244758:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320290.244761:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5534 00000100:00080000:3.0:1767320290.244762:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320290.244765:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320290.244775:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3100 x1853163983032576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320290.244791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320290.244792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320290.244793:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320290.244796:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5534 00000100:00080000:3.0:1767320290.244798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320290.244800:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320290.244806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0380 x1853163983032832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320290.244811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320290.244813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320290.244814:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320290.244815:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5534 00000100:00080000:3.0:1767320290.244817:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320290.244818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320290.244824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0000 x1853163983032960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320290.244835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320290.244837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320290.244838:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320290.244839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5534 00000100:00080000:3.0:1767320290.244841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320290.244842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320290.244847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163983033344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320290.244862:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983032704:0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.244913:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320290.244969:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320290.244975:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983033088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.245014:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320290.245097:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983033216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320290.245125:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320290.245158:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320290.245164:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983032448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320290.245183:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320290.245213:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032704 00000100:00100000:2.0:1767320290.245220:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032704 00000100:00100000:2.0:1767320290.245265:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032448 00000100:00100000:2.0:1767320290.245299:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032448 00000100:00100000:2.0:1767320290.245321:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1625 00000100:00100000:2.0:1767320290.245333:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983032704:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320290.245386:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983032704:12345-0@lo:400:kworker.0 Request processed in 57us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320290.245393:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1625 00000100:00100000:2.0:1767320290.245408:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1626 00000100:00100000:2.0:1767320290.245414:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983032448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320290.245445:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983032448:12345-0@lo:400:kworker.0 Request processed in 31us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320290.245451:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1626 00000100:00080000:2.0:1767320290.245466:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320290.245490:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2a00 x1853163983032448/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320306 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320290.245511:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983032448:0@lo:400:kworker.0 00010000:00080000:2.0:1767320290.245533:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1f80 x1853163983032704/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320306 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320290.245541:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983032704:0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.245674:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033088 00000100:00100000:3.0:1767320290.245678:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033088 00000100:00100000:3.0:1767320290.245711:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033216 00000100:00100000:3.0:1767320290.245713:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033216 00000100:00100000:3.0:1767320290.245729:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011 00000100:00100000:3.0:1767320290.245735:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983033088:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.245766:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983033088:12345-0@lo:400:kworker.0 Request processed in 31us (751us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320290.245771:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011 00000100:00100000:3.0:1767320290.245780:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1012 00000100:00100000:3.0:1767320290.245785:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983033216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.245809:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983033216:12345-0@lo:400:kworker.0 Request processed in 24us (683us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320290.245814:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1012 00000100:00100000:3.0:1767320290.245836:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983033216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320290.245851:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983033088:0@lo:400:kworker.0 00000100:00080000:2.0:1767320290.245895:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320290.245898:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320290.246304:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983032320:0@lo:250:kworker.0 00000400:00000100:0.0:1767320290.246330:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983032320 offset 0 length 520: 4 00000100:00100000:0.0:1767320290.246351:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983032576:0@lo:38:kworker.0 00000100:00100000:0.0:1767320290.246371:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320290.246399:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983032832:0@lo:38:kworker.0 00000100:00100000:0.0:1767320290.246419:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320290.246443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983032960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320290.246466:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320290.246489:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983033344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320290.246541:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032576 00000100:00100000:2.0:1767320290.246545:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032576 00000100:00100000:2.0:1767320290.246563:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032832 00000100:00100000:2.0:1767320290.246565:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032832 00000100:00100000:2.0:1767320290.246577:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148 00000100:00100000:2.0:1767320290.246581:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983032576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320290.246597:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320290.246629:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983032576:12345-0@lo:38:kworker.0 Request processed in 47us (257us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320290.246635:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148 00000100:00100000:2.0:1767320290.246645:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2149 00000100:00100000:2.0:1767320290.246648:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983032832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320290.246656:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320290.246670:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983032832:12345-0@lo:38:kworker.0 Request processed in 21us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320290.246674:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2149 00000100:00100000:0.0:1767320290.246745:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320290.246776:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320290.246781:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320290.246784:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983032576:0@lo:38:kworker.0 00000100:00100000:1.0:1767320290.246812:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033344 00000100:00100000:1.0:1767320290.246816:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033344 00000100:00100000:1.0:1767320290.246840:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2150 00000100:00100000:1.0:1767320290.246844:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983033344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320290.246853:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320290.246876:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983033344:12345-0@lo:38:kworker.0 Request processed in 32us (368us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320290.246881:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2150 00000100:00080000:0.0:1767320290.246948:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320290.246952:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320290.246955:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983032832:0@lo:38:kworker.0 00000100:00080000:0.0:1767320290.246963:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320290.246966:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320290.246969:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983033344:0@lo:38:kworker.0 00000100:00100000:3.0:1767320290.247431:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983032960 00000100:00100000:3.0:1767320290.247437:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983032960 00000100:00100000:3.0:1767320290.247472:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 562 00000100:00100000:3.0:1767320290.247478:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983032960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320290.247489:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320290.247542:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983032960:12345-0@lo:38:kworker.0 Request processed in 64us (1076us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320290.247548:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 562 00000100:00080000:1.0:1767320290.247608:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320290.247612:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320290.247615:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983032960:0@lo:38:kworker.0 00010000:00010000:1.0:1767320290.436989:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320291.460805:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320292.240846:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320292.241016:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320292.241950:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673216 00000100:00100000:3.0:1767320292.241975:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673216 00000100:00100000:3.0:1767320292.242126:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673088 00000100:00100000:3.0:1767320292.242131:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673088 00000100:00100000:3.0:1767320292.242156:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1627 00000100:00100000:3.0:1767320292.242176:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320292.242536:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 370us (1701us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320292.242545:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1627 00000100:00100000:3.0:1767320292.242852:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2151 00000100:00100000:3.0:1767320292.242861:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915673088:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320292.242884:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320292.242915:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915673088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 54us (1898us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320292.242921:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2151 00000100:00080000:0.0:1767320292.244053:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320292.485813:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320293.508595:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320293.828753:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983033472:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320293.828822:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320293.829028:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033472 00000100:00100000:0.0:1767320293.829037:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033472 00000100:00100000:0.0:1767320293.829108:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092 00000100:00080000:1.0:1767320293.829263:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320293.829434:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983033472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320293.829466:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320293.829526:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983033472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (703us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320293.829597:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983033472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320293.830029:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320293.830041:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983033600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320293.830075:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320293.830628:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033600 00000100:00100000:1.0:1767320293.830635:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033600 00000100:00080000:2.0:1767320293.830709:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320293.837675:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092 00000100:00100000:1.0:1767320293.838711:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1093 00000100:00100000:1.0:1767320293.838732:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983033600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320293.838767:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320293.838902:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983033600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 180us (8825us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320293.838910:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1093 00000100:00100000:2.0:1767320293.840793:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983033600:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320294.532642:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320295.364402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320295.364414:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320295.364502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320295.364507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320295.364511:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320295.364519:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5538 00000100:00080000:3.0:1767320295.364524:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320295.364532:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320295.364555:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1c00 x1853163983033856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320295.364602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320295.364605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320295.364607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320295.364611:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5538 00000100:00080000:3.0:1767320295.364614:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320295.364618:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320295.364628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1180 x1853163983034112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320295.364636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320295.364639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320295.364640:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320295.364643:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5538 00000100:00080000:3.0:1767320295.364646:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320295.364649:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320295.364658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2d80 x1853163983034240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320295.364678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320295.364682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320295.364684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320295.364687:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5538 00000100:00080000:3.0:1767320295.364689:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320295.364692:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320295.364703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3480 x1853163983034624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320295.364714:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983033856:0@lo:38:kworker.0 00000100:00100000:3.0:1767320295.364734:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983033984:0@lo:400:kworker.0 00000100:00100000:0.0:1767320295.364981:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983034496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320295.365052:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983033728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320295.369575:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320295.369636:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983034112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320295.369674:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320295.369694:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983034240:0@lo:38:kworker.0 00000100:00100000:2.0:1767320295.369716:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320295.369745:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983034624:0@lo:38:kworker.0 00000100:00100000:2.0:1767320295.369766:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320295.369893:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033856 00000100:00100000:2.0:1767320295.369901:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033856 00000100:00100000:2.0:1767320295.370711:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034112 00000100:00100000:2.0:1767320295.370718:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034112 00000100:00100000:2.0:1767320295.370733:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034624 00000100:00100000:2.0:1767320295.370736:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034624 00000100:00100000:2.0:1767320295.370748:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767320295.370755:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983033856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320295.370778:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320295.371458:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983033856:12345-0@lo:38:kworker.0 Request processed in 627us (1816us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320295.371470:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767320295.371491:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2153 00000100:00100000:2.0:1767320295.371496:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983034112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320295.371509:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320295.371537:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983034112:12345-0@lo:38:kworker.0 Request processed in 40us (1861us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320295.371543:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2153 00000100:00100000:2.0:1767320295.371553:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154 00000100:00100000:2.0:1767320295.371558:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983034624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320295.371567:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320295.371591:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983034624:12345-0@lo:38:kworker.0 Request processed in 31us (1823us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320295.371596:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154 00000100:00080000:2.0:1767320295.371647:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320295.371653:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320295.371656:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983034624:0@lo:38:kworker.0 00000100:00080000:2.0:1767320295.371667:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320295.371671:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320295.371673:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983033856:0@lo:38:kworker.0 00000100:00080000:2.0:1767320295.371681:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320295.371683:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320295.371686:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983034112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320295.371754:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034240 00000100:00100000:2.0:1767320295.371758:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034240 00000100:00100000:2.0:1767320295.371787:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 563 00000100:00100000:2.0:1767320295.371792:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983034240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320295.371896:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320295.372100:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983034240:12345-0@lo:38:kworker.0 Request processed in 305us (2381us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320295.372108:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 563 00000100:00080000:2.0:1767320295.372132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320295.372136:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320295.372140:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983034240:0@lo:38:kworker.0 00000100:00100000:2.0:1767320295.372372:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983034368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320295.372486:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320295.372536:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034368 00000100:00100000:2.0:1767320295.372540:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034368 00000100:00100000:2.0:1767320295.372587:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013 00000100:00100000:2.0:1767320295.372594:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983034368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320295.372636:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983034368:12345-0@lo:400:kworker.0 Request processed in 43us (150us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320295.372643:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013 00000100:00080000:2.0:1767320295.372660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320295.372678:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983034368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320295.373232:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320295.373395:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320295.373460:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034496 00000100:00100000:0.0:1767320295.373467:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034496 00000100:00100000:0.0:1767320295.373611:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1014 00000100:00100000:0.0:1767320295.373620:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983034496:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320295.373659:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983034496:12345-0@lo:400:kworker.0 Request processed in 41us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320295.373665:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1014 00000100:00100000:2.0:1767320295.373717:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033984 00000100:00100000:2.0:1767320295.373722:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033984 00000100:00100000:2.0:1767320295.373748:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1628 00000100:00100000:2.0:1767320295.373755:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983033984:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320295.373804:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983033984:12345-0@lo:400:kworker.0 Request processed in 50us (574us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320295.373811:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1628 00000100:00100000:1.0:1767320295.373892:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320295.374071:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983033728 00000100:00100000:1.0:1767320295.374075:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983033728 00000100:00100000:1.0:1767320295.374104:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1629 00000100:00100000:1.0:1767320295.374111:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983033728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320295.374142:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983033728:12345-0@lo:400:kworker.0 Request processed in 32us (250us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320295.374147:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1629 00010000:00080000:1.0:1767320295.374181:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b3800 x1853163983033728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320311 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320295.374194:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983033728:0@lo:400:kworker.0 00000100:00080000:1.0:1767320295.374218:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320295.374222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320295.374240:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983034496:0@lo:400:kworker.0 00010000:00080000:3.0:1767320295.376954:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b3b80 x1853163983033984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320311 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320295.376975:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983033984:0@lo:400:kworker.0 00010000:00010000:2.0:1767320295.559967:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320296.580606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320297.357091:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320297.357220:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673472 00000100:00100000:1.0:1767320297.357233:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673472 00000100:00100000:1.0:1767320297.357360:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1630 00000100:00100000:1.0:1767320297.357372:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915673472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320297.357457:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915673472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320297.357465:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1630 00000100:00080000:2.0:1767320297.358106:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320297.361072:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320297.362983:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673344 00000100:00100000:3.0:1767320297.363003:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673344 00000100:00100000:3.0:1767320297.363077:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2155 00000100:00100000:3.0:1767320297.363087:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915673344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320297.363110:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320297.363197:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915673344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 109us (2134us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320297.363211:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2155 00010000:00010000:2.0:1767320297.604472:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320298.628523:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320298.948725:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320298.949144:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983034880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320298.949207:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320298.949453:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034880 00000100:00100000:0.0:1767320298.949461:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034880 00000100:00100000:0.0:1767320298.949518:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094 00000100:00100000:0.0:1767320298.949527:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983034880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320298.949554:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320298.949602:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983034880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320298.949609:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094 00000100:00100000:0.0:1767320298.949661:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983034880:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320298.949780:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320298.950473:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983034752:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320298.950514:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320298.950609:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983034752 00000100:00100000:1.0:1767320298.950616:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983034752 00000100:00100000:1.0:1767320298.950670:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095 00000100:00100000:1.0:1767320298.950677:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983034752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320298.950699:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320298.950747:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983034752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320298.950754:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767320298.950910:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983034752:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320298.950970:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320299.653122:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320300.484498:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320300.484509:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320300.484666:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320300.484671:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320300.484674:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320300.484683:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5543 00000100:00080000:1.0:1767320300.484689:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320300.484696:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320300.484702:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320300.484717:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983035008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320300.484719:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab100 x1853163983035136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320300.484759:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320300.484762:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320300.484764:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320300.484768:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5543 00000100:00080000:1.0:1767320300.484770:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320300.484773:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320300.484789:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8380 x1853163983035392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320300.484798:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320300.484802:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320300.484803:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320300.484806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5543 00000100:00080000:1.0:1767320300.484808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320300.484823:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320300.484835:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8000 x1853163983035520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320300.484842:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320300.484862:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320300.484865:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320300.484867:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320300.484869:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5543 00000100:00080000:1.0:1767320300.484872:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320300.484875:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320300.484879:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983035776:0@lo:400:kworker.0 00000100:00100000:1.0:1767320300.484883:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa680 x1853163983035904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320300.484905:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983035648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320300.484924:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320300.484937:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035008 00000100:00100000:1.0:1767320300.484944:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035008 00000100:00100000:2.0:1767320300.484956:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035776 00000100:00100000:2.0:1767320300.484959:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035776 00000100:00100000:1.0:1767320300.484990:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1631 00000100:00100000:2.0:1767320300.484990:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1015 00000100:00100000:2.0:1767320300.484998:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983035776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320300.484999:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983035008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320300.485041:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983035776:12345-0@lo:400:kworker.0 Request processed in 43us (115us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320300.485048:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1015 00000100:00080000:2.0:1767320300.485066:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320300.485070:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320300.485078:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320300.485078:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983035136:0@lo:38:kworker.0 00000100:00100000:2.0:1767320300.485096:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320300.485101:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983035264:0@lo:400:kworker.0 00000100:00100000:3.0:1767320300.485105:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320300.485120:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320300.485120:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983035392:0@lo:38:kworker.0 00000100:00100000:3.0:1767320300.485137:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320300.485156:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983035520:0@lo:38:kworker.0 00000100:00100000:3.0:1767320300.485176:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320300.485228:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983035904:0@lo:38:kworker.0 00000100:00100000:3.0:1767320300.485248:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320300.485249:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035136 00000100:00100000:2.0:1767320300.485253:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035136 00000100:00100000:2.0:1767320300.485266:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035264 00000100:00100000:2.0:1767320300.485284:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035264 00000100:00100000:3.0:1767320300.485287:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035392 00000100:00100000:3.0:1767320300.485290:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035392 00000100:00100000:2.0:1767320300.485302:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035904 00000100:00100000:2.0:1767320300.485304:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035904 00000100:00100000:3.0:1767320300.485304:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632 00000100:00100000:3.0:1767320300.485310:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983035264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320300.485313:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2156 00000100:00100000:2.0:1767320300.485317:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983035136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320300.485334:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320300.485340:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983035264:12345-0@lo:400:kworker.0 Request processed in 31us (220us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320300.485346:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632 00000100:00100000:3.0:1767320300.485358:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157 00000100:00100000:2.0:1767320300.485362:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983035136:12345-0@lo:38:kworker.0 Request processed in 44us (256us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320300.485362:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983035392:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320300.485368:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2156 00010000:00020000:3.0:1767320300.485371:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320300.485377:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2158 00000100:00100000:2.0:1767320300.485381:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983035904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320300.485389:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320300.485390:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983035392:12345-0@lo:38:kworker.0 Request processed in 27us (252us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320300.485395:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157 00000100:00100000:2.0:1767320300.485406:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983035904:12345-0@lo:38:kworker.0 Request processed in 24us (158us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320300.485411:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2158 00010000:00080000:3.0:1767320300.485423:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9f80 x1853163983035264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320316 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320300.485427:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035520 00000100:00100000:2.0:1767320300.485430:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035520 00000100:00100000:3.0:1767320300.485436:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983035264:0@lo:400:kworker.0 00000100:00100000:2.0:1767320300.485446:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 564 00000100:00100000:2.0:1767320300.485450:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983035520:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320300.485458:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320300.485460:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320300.485465:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320300.485468:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983035136:0@lo:38:kworker.0 00000100:00080000:3.0:1767320300.485475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320300.485478:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320300.485479:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983035520:12345-0@lo:38:kworker.0 Request processed in 28us (302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320300.485482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983035392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320300.485485:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 564 00000100:00080000:2.0:1767320300.485503:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320300.485504:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320300.485507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320300.485509:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983035648 00000100:00100000:3.0:1767320300.485510:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983035520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320300.485512:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983035648 00000100:00080000:3.0:1767320300.485516:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320300.485519:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320300.485522:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983035904:0@lo:38:kworker.0 00000100:00100000:2.0:1767320300.485534:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016 00000100:00100000:2.0:1767320300.485540:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983035648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320300.485568:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983035648:12345-0@lo:400:kworker.0 Request processed in 28us (488us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320300.485574:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016 00000100:00100000:1.0:1767320300.486172:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983035008:12345-0@lo:400:kworker.0 Request processed in 1172us (1328us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320300.486190:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1631 00000100:00080000:1.0:1767320300.486220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320300.486248:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983035648:0@lo:400:kworker.0 00000100:00100000:3.0:1767320300.492759:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983035776:0@lo:400:kworker.0 00010000:00080000:3.0:1767320300.492798:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aaa00 x1853163983035008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320316 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320300.492809:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983035008:0@lo:400:kworker.0 00010000:00010000:2.0:1767320300.676509:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320301.702037:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320302.482353:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320302.482446:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673600 00000100:00100000:1.0:1767320302.482457:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673600 00000100:00100000:1.0:1767320302.482507:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2159 00000100:00100000:1.0:1767320302.482516:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915673600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320302.482539:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320302.482621:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915673600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 104us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320302.482630:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2159 00000100:00100000:0.0:1767320302.487156:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320302.488369:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673728 00000100:00100000:2.0:1767320302.488384:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673728 00000100:00100000:2.0:1767320302.488557:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1633 00000100:00100000:2.0:1767320302.488567:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320302.488646:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (1501us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320302.488654:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1633 00000100:00080000:1.0:1767320302.493806:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320302.725482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320303.748630:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320304.068735:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320304.068780:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320304.068790:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983036160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320304.068795:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983036032:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320304.076826:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320304.077363:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036032 00000100:00100000:1.0:1767320304.077373:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036032 00000100:00100000:1.0:1767320304.077447:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096 00000100:00100000:1.0:1767320304.077457:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983036032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320304.077493:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320304.077563:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983036032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 109us (746us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320304.077572:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096 00000100:00080000:3.0:1767320304.079380:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320304.079455:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983036032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320304.085495:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320304.085733:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036160 00000100:00100000:1.0:1767320304.085743:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036160 00000100:00100000:1.0:1767320304.085828:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097 00000100:00100000:1.0:1767320304.085838:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983036160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320304.085871:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320304.085936:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983036160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 100us (449us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320304.085945:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097 00000100:00080000:3.0:1767320304.085953:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320304.086093:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983036160:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320304.773133:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320305.604639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320305.604650:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320305.604710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320305.604715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320305.604746:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320305.604755:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5549 00000100:00080000:0.0:1767320305.604760:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320305.604767:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320305.604790:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3480 x1853163983036416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320305.604846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320305.604849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320305.604851:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320305.604854:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5549 00000100:00080000:0.0:1767320305.604857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320305.604860:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320305.604872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0380 x1853163983036672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320305.604881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320305.604884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320305.604886:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320305.604889:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5549 00000100:00080000:0.0:1767320305.604892:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320305.604896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320305.604908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1500 x1853163983036800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320305.604928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320305.604932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320305.604933:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320305.604937:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5549 00000100:00080000:0.0:1767320305.604939:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320305.604943:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320305.604963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0000 x1853163983037184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320305.604993:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983036928:0@lo:400:kworker.0 00000100:00100000:0.0:1767320305.605054:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320305.605111:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320305.605117:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983036544:0@lo:400:kworker.0 00000100:00100000:0.0:1767320305.605143:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320305.605218:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983036416:0@lo:38:kworker.0 00000100:00100000:1.0:1767320305.605247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320305.605300:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983036672:0@lo:38:kworker.0 00000100:00100000:1.0:1767320305.605322:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320305.605338:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983036800:0@lo:38:kworker.0 00000100:00100000:1.0:1767320305.605360:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320305.605379:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983037184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320305.605398:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320305.605483:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983037056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320305.605512:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320305.605541:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320305.605547:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983036288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320305.605567:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320305.605599:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036544 00000100:00100000:3.0:1767320305.605608:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036544 00000100:00100000:3.0:1767320305.605648:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036416 00000100:00100000:3.0:1767320305.605651:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036416 00000100:00100000:3.0:1767320305.605659:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036672 00000100:00100000:3.0:1767320305.605661:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036672 00000100:00100000:3.0:1767320305.605667:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037184 00000100:00100000:3.0:1767320305.605669:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037184 00000100:00100000:3.0:1767320305.605675:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036288 00000100:00100000:3.0:1767320305.605677:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036288 00000100:00100000:3.0:1767320305.605708:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1634 00000100:00100000:3.0:1767320305.605716:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983036544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320305.605773:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983036544:12345-0@lo:400:kworker.0 Request processed in 57us (628us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320305.605780:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1634 00000100:00100000:3.0:1767320305.605798:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1635 00000100:00100000:3.0:1767320305.605804:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983036288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320305.605845:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983036288:12345-0@lo:400:kworker.0 Request processed in 30us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320305.605852:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1635 00000100:00100000:3.0:1767320305.605865:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160 00000100:00100000:3.0:1767320305.605869:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983036416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320305.605886:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320305.605918:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983036416:12345-0@lo:38:kworker.0 Request processed in 47us (670us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320305.605924:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160 00000100:00100000:3.0:1767320305.605936:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2161 00000100:00100000:3.0:1767320305.605940:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983036672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320305.605948:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320305.605970:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983036672:12345-0@lo:38:kworker.0 Request processed in 28us (647us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320305.605975:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2161 00000100:00100000:3.0:1767320305.605986:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2162 00000100:00100000:3.0:1767320305.605990:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983037184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320305.605999:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320305.606018:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983037184:12345-0@lo:38:kworker.0 Request processed in 27us (620us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320305.606024:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2162 00000100:00080000:3.0:1767320305.606042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320305.606068:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2a00 x1853163983036288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320321 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320305.606081:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983036288:0@lo:400:kworker.0 00000100:00080000:3.0:1767320305.606106:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320305.606110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320305.606113:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983036416:0@lo:38:kworker.0 00000100:00080000:3.0:1767320305.606120:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320305.606123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320305.606125:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983036672:0@lo:38:kworker.0 00000100:00080000:3.0:1767320305.606132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320305.606135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320305.606138:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983037184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320305.606488:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036928 00000100:00100000:1.0:1767320305.606494:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036928 00000100:00100000:1.0:1767320305.606530:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037056 00000100:00100000:1.0:1767320305.606534:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037056 00000100:00100000:1.0:1767320305.606553:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017 00000100:00100000:1.0:1767320305.606560:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983036928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320305.606597:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983036928:12345-0@lo:400:kworker.0 Request processed in 38us (1544us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320305.606603:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017 00000100:00100000:1.0:1767320305.606616:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1018 00000100:00100000:1.0:1767320305.606621:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983037056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320305.606648:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983037056:12345-0@lo:400:kworker.0 Request processed in 27us (1135us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320305.606653:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1018 00010000:00080000:0.0:1767320305.607035:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f3800 x1853163983036544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320321 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320305.607047:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983036544:0@lo:400:kworker.0 00000100:00100000:0.0:1767320305.607058:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983036928:0@lo:400:kworker.0 00000100:00100000:2.0:1767320305.608478:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983036800 00000100:00100000:2.0:1767320305.608489:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983036800 00000100:00100000:2.0:1767320305.608527:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 565 00000100:00100000:2.0:1767320305.608534:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983036800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320305.608554:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320305.608628:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983036800:12345-0@lo:38:kworker.0 Request processed in 93us (3267us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320305.608635:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 565 00000100:00100000:3.0:1767320305.608781:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983037056:0@lo:400:kworker.0 00000100:00080000:1.0:1767320305.612116:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320305.612129:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320305.612135:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983036800:0@lo:38:kworker.0 00010000:00010000:1.0:1767320305.796551:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320306.822822:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320307.597794:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320307.597839:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320307.598233:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673984 00000100:00100000:2.0:1767320307.598246:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673984 00000100:00100000:1.0:1767320307.598407:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915673856 00000100:00100000:1.0:1767320307.598412:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915673856 00000100:00080000:3.0:1767320307.598542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320307.599309:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1636 00000100:00100000:1.0:1767320307.599325:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320307.599412:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915673984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (1628us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320307.599424:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1636 00000100:00100000:1.0:1767320307.599442:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2163 00000100:00100000:1.0:1767320307.599446:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915673856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320307.599465:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320307.599489:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915673856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (1649us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320307.599495:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2163 00010000:00010000:2.0:1767320307.845364:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320308.868841:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320309.190600:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320309.190622:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983037312:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320309.190697:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320309.190837:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037312 00000100:00100000:0.0:1767320309.190844:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037312 00000100:00100000:0.0:1767320309.190986:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098 00000100:00100000:0.0:1767320309.190996:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983037312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320309.191026:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320309.191081:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983037312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (383us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320309.191088:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098 00000100:00100000:2.0:1767320309.191136:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983037312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320309.193129:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983037440:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320309.193187:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320309.193291:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037440 00000100:00100000:0.0:1767320309.193299:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037440 00000100:00080000:3.0:1767320309.196723:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320309.196732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320309.205782:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099 00000100:00100000:0.0:1767320309.205808:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983037440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320309.205846:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320309.205939:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983037440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 144us (12751us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320309.205947:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099 00000100:00100000:3.0:1767320309.206066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983037440:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320309.894219:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320310.725120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320310.725132:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320310.725192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320310.725196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320310.725199:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320310.725207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5554 00000100:00080000:3.0:1767320310.725211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320310.725218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320310.725239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163983037696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320310.725282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320310.725285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320310.725287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320310.725290:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5554 00000100:00100000:0.0:1767320310.725292:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983037568:0@lo:400:kworker.0 00000100:00080000:3.0:1767320310.725293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320310.725296:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320310.725308:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1500 x1853163983037952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320310.725320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320310.725323:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320310.725325:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320310.725327:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5554 00000100:00080000:3.0:1767320310.725330:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320310.725333:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320310.725342:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163983038080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320310.725359:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320310.725359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320310.725362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320310.725364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320310.725367:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5554 00000100:00080000:3.0:1767320310.725369:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320310.725372:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320310.725380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0000 x1853163983038464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320310.725419:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037568 00000100:00100000:0.0:1767320310.725420:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320310.725426:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983038336:0@lo:400:kworker.0 00000100:00100000:3.0:1767320310.725428:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037568 00000100:00100000:0.0:1767320310.725452:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320310.725477:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038336 00000100:00100000:3.0:1767320310.725480:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1637 00000100:00100000:0.0:1767320310.725481:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038336 00000100:00100000:3.0:1767320310.725489:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983037568:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320310.725506:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019 00000100:00100000:0.0:1767320310.725511:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983038336:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320310.725538:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983038336:12345-0@lo:400:kworker.0 Request processed in 27us (85us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320310.725538:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983037568:12345-0@lo:400:kworker.0 Request processed in 50us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320310.725545:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019 00000100:00100000:3.0:1767320310.725545:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1637 00000100:00080000:3.0:1767320310.725565:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320310.725585:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983038336:0@lo:400:kworker.0 00010000:00080000:0.0:1767320310.725596:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b3100 x1853163983037568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320326 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320310.725605:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983037568:0@lo:400:kworker.0 00000100:00100000:2.0:1767320310.725639:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983037824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320310.725644:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983038208:0@lo:400:kworker.0 00000100:00100000:0.0:1767320310.725666:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320310.725666:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320310.725698:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037824 00000100:00100000:0.0:1767320310.725702:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037824 00000100:00100000:3.0:1767320310.725738:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038208 00000100:00100000:0.0:1767320310.725740:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1767320310.725742:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038208 00000100:00100000:0.0:1767320310.725746:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983037824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320310.725762:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020 00000100:00100000:3.0:1767320310.725767:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983038208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320310.725775:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983037824:12345-0@lo:400:kworker.0 Request processed in 30us (109us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320310.725781:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1767320310.725797:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983038208:12345-0@lo:400:kworker.0 Request processed in 30us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320310.725803:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983038208:0@lo:400:kworker.0 00000100:00100000:3.0:1767320310.725803:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020 00000100:00080000:2.0:1767320310.725864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320310.725881:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2a00 x1853163983037824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320326 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320310.725891:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983037824:0@lo:400:kworker.0 00000100:00100000:1.0:1767320310.726086:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983037696:0@lo:38:kworker.0 00000100:00100000:1.0:1767320310.726116:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320310.726149:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983037952:0@lo:38:kworker.0 00000100:00100000:1.0:1767320310.726171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320310.726193:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983038080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320310.726218:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320310.726245:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983038464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320310.726264:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320310.726326:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037696 00000100:00100000:3.0:1767320310.726331:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037696 00000100:00100000:3.0:1767320310.726358:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2164 00000100:00100000:3.0:1767320310.726364:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983037696:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320310.726377:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038080 00010000:00020000:3.0:1767320310.726380:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320310.726381:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038080 00000100:00100000:0.0:1767320310.726402:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 566 00000100:00100000:0.0:1767320310.726406:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93450725d050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983038080:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320310.726410:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983037696:12345-0@lo:38:kworker.0 Request processed in 46us (293us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320310.726415:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2164 00010000:00020000:0.0:1767320310.726416:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320310.726424:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983037952 00000100:00100000:3.0:1767320310.726427:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983037952 00000100:00100000:0.0:1767320310.726438:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93450725d050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983038080:12345-0@lo:38:kworker.0 Request processed in 31us (220us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320310.726444:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 566 00000100:00100000:2.0:1767320310.727304:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038464 00000100:00100000:2.0:1767320310.727312:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038464 00000100:00100000:2.0:1767320310.727345:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2165 00000100:00100000:2.0:1767320310.727351:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983037952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320310.727367:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320310.727411:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983037952:12345-0@lo:38:kworker.0 Request processed in 58us (1238us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320310.727417:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2165 00000100:00100000:2.0:1767320310.727430:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166 00000100:00100000:2.0:1767320310.727434:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983038464:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320310.727443:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320310.727467:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983038464:12345-0@lo:38:kworker.0 Request processed in 32us (1201us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320310.727472:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166 00000100:00080000:1.0:1767320310.727754:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320310.727759:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320310.727763:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983037696:0@lo:38:kworker.0 00000100:00080000:1.0:1767320310.727772:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320310.727776:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320310.727778:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983037952:0@lo:38:kworker.0 00000100:00080000:1.0:1767320310.727787:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320310.727790:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320310.727792:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983038080:0@lo:38:kworker.0 00000100:00080000:1.0:1767320310.727799:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320310.727802:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320310.727804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983038464:0@lo:38:kworker.0 00010000:00010000:2.0:1767320310.920897:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320311.940878:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320312.716413:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767320312.716459:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915674112 offset 0 length 520: 4 00000100:00100000:0.0:1767320312.716499:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320312.719164:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674368 00000100:00100000:1.0:1767320312.719181:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674368 00000100:00100000:1.0:1767320312.719286:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674240 00000100:00100000:1.0:1767320312.719290:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674240 00000100:00100000:1.0:1767320312.719319:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1639 00000100:00100000:1.0:1767320312.719331:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915674368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320312.719434:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915674368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 105us (3032us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320312.719442:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1639 00000100:00100000:1.0:1767320312.719465:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2167 00000100:00100000:1.0:1767320312.719470:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915674240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320312.719489:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320312.719516:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915674240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (3016us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320312.719523:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2167 00000100:00080000:3.0:1767320312.719666:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320312.968431:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320313.988684:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320314.309493:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320314.309523:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983038592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320314.309632:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320314.309948:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038592 00000100:00100000:0.0:1767320314.309959:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038592 00000100:00100000:0.0:1767320314.310026:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100 00000100:00100000:0.0:1767320314.310037:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983038592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320314.310072:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320314.310119:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983038592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (486us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320314.310126:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100 00000100:00100000:0.0:1767320314.310752:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983038720:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320314.310798:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320314.318977:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038720 00000100:00100000:1.0:1767320314.318995:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038720 00000100:00100000:1.0:1767320314.319076:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101 00000100:00100000:1.0:1767320314.319087:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983038720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320314.319125:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320314.319208:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983038720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (8409us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320314.319216:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101 00000100:00080000:3.0:1767320314.319461:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320314.319468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320314.319535:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983038592:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320314.320608:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983038720:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320315.012540:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320315.844384:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320290/real 1767320290] req@ffff93460469ca80 x1853163983032320/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320315 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320315.844413:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460469ca80 x1853163983032320/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320315 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320315.844428:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320315.844433:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320315.844440:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320315.844448:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320315.844504:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320315.844509:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983032320:0@lo:250:kworker.0 00000100:00080000:3.0:1767320315.844668:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5538 00000100:00080000:3.0:1767320315.844672:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320315.844679:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320315.844713:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0380 x1853163983038848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320315.844782:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983038848:0@lo:250:kworker.0 00000400:00000100:2.0:1767320315.844833:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983038848 offset 0 length 520: 4 00000100:00100000:1.0:1767320315.844947:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983038976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320315.844988:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320315.845087:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320315.845091:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320315.845094:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320315.845097:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5559 00000100:00080000:3.0:1767320315.845100:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320315.845104:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320315.845122:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2a00 x1853163983039104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320315.845154:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983039104:0@lo:38:kworker.0 00000100:00100000:2.0:1767320315.845179:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320315.845452:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983039232:0@lo:400:kworker.0 00000100:00080000:3.0:1767320315.845673:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320315.845678:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320315.845680:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320315.845685:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5559 00000100:00080000:3.0:1767320315.845687:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320315.845692:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320315.845707:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163983039360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320315.845717:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320315.845720:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320315.845722:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320315.845724:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5559 00000100:00080000:3.0:1767320315.845727:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320315.845730:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320315.845740:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1500 x1853163983039488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320315.845789:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983039616:0@lo:400:kworker.0 00000100:00100000:1.0:1767320315.845820:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320315.845909:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983039360:0@lo:38:kworker.0 00000100:00100000:2.0:1767320315.845931:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320315.845965:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320315.845969:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320315.845971:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320315.845975:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5559 00000100:00080000:3.0:1767320315.845977:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320315.845981:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320315.845993:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3800 x1853163983039872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320315.846015:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320315.846023:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983039744:0@lo:400:kworker.0 00000100:00100000:3.0:1767320315.846050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320315.846076:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039616 00000100:00100000:3.0:1767320315.846083:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039616 00000100:00100000:3.0:1767320315.846123:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039744 00000100:00100000:3.0:1767320315.846126:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039744 00000100:00100000:3.0:1767320315.846145:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1021 00000100:00100000:3.0:1767320315.846153:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983039616:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320315.846212:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983038976 00000100:00100000:0.0:1767320315.846217:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983038976 00000100:00100000:0.0:1767320315.846252:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039104 00000100:00100000:0.0:1767320315.846255:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039104 00000100:00100000:0.0:1767320315.846287:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1640 00000100:00100000:0.0:1767320315.846295:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983038976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320315.846334:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983038976:12345-0@lo:400:kworker.0 Request processed in 40us (1345us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320315.846341:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1640 00000100:00100000:0.0:1767320315.846358:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2168 00000100:00100000:0.0:1767320315.846362:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983039104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320315.846378:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320315.846405:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983039104:12345-0@lo:38:kworker.0 Request processed in 42us (1225us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320315.846411:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2168 00000100:00100000:0.0:1767320315.846444:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1022 00000100:00100000:0.0:1767320315.846451:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983039744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320315.846487:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983039744:12345-0@lo:400:kworker.0 Request processed in 28us (427us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320315.846492:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1022 00000100:00080000:0.0:1767320315.846516:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320315.846553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983039744:0@lo:400:kworker.0 00010000:00080000:0.0:1767320315.846566:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f3480 x1853163983038976/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320331 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320315.846575:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983038976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320315.847100:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983039488:0@lo:38:kworker.0 00000100:00100000:2.0:1767320315.847133:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320315.847158:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320315.847163:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320315.847166:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983039104:0@lo:38:kworker.0 00000100:00100000:2.0:1767320315.847182:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983039872:0@lo:38:kworker.0 00000100:00100000:2.0:1767320315.847210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320315.847240:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039488 00000100:00100000:2.0:1767320315.847245:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039488 00000100:00100000:2.0:1767320315.847285:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 567 00000100:00100000:2.0:1767320315.847291:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983039488:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320315.847304:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320315.847336:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983039488:12345-0@lo:38:kworker.0 Request processed in 44us (202us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320315.847343:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 567 00000100:00100000:2.0:1767320315.847364:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039360 00000100:00100000:2.0:1767320315.847368:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039360 00000100:00100000:2.0:1767320315.847385:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039872 00000100:00100000:2.0:1767320315.847388:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039872 00000100:00100000:2.0:1767320315.847404:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169 00000100:00100000:2.0:1767320315.847408:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983039360:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320315.847418:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320315.847443:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983039360:12345-0@lo:38:kworker.0 Request processed in 33us (1511us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320315.847448:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169 00000100:00100000:2.0:1767320315.847459:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2170 00000100:00100000:2.0:1767320315.847463:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983039872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320315.847472:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320315.847509:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983039872:12345-0@lo:38:kworker.0 Request processed in 44us (298us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320315.847516:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2170 00000100:00100000:2.0:1767320315.847548:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320315.847577:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983039232 00000100:00100000:2.0:1767320315.847581:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983039232 00000100:00100000:2.0:1767320315.847611:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1641 00000100:00100000:2.0:1767320315.847618:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983039232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320315.847658:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983039232:12345-0@lo:400:kworker.0 Request processed in 40us (109us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320315.847664:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1641 00000100:00080000:2.0:1767320315.847679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320315.847697:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f0000 x1853163983039232/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320331 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320315.847708:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983039232:0@lo:400:kworker.0 00000100:00080000:2.0:1767320315.847732:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320315.847736:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320315.847739:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983039872:0@lo:38:kworker.0 00000100:00080000:2.0:1767320315.847748:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320315.847752:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320315.847754:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983039360:0@lo:38:kworker.0 00000100:00080000:2.0:1767320315.847761:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320315.847764:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320315.847767:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983039488:0@lo:38:kworker.0 00000100:00100000:3.0:1767320315.848040:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983039616:12345-0@lo:400:kworker.0 Request processed in 1886us (2218us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320315.848050:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1021 00000100:00100000:1.0:1767320315.848246:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983039616:0@lo:400:kworker.0 00010000:00010000:2.0:1767320316.036503:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320317.060519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320317.839732:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320317.839833:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674624 00000100:00100000:0.0:1767320317.839844:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674624 00000100:00100000:0.0:1767320317.839977:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1642 00000100:00100000:0.0:1767320317.839990:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915674624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320317.840088:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915674624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320317.840096:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1642 00000100:00080000:2.0:1767320317.840904:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320317.841720:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320317.841805:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674496 00000100:00100000:1.0:1767320317.841814:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674496 00000100:00100000:1.0:1767320317.841860:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2171 00000100:00100000:1.0:1767320317.841867:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915674496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320317.841888:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320317.841952:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915674496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (236us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320317.841960:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2171 00010000:00010000:1.0:1767320318.084550:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320319.110834:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320319.428586:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983040000:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320319.428667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320319.428757:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040000 00000100:00100000:0.0:1767320319.428865:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983040128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320319.428900:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320319.428938:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040128 00000100:00100000:0.0:1767320319.428945:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040128 00000100:00080000:2.0:1767320319.428974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320319.429006:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1102 00000100:00100000:0.0:1767320319.429016:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324c0450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983040128:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320319.429048:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320319.429102:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324c0450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983040128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (201us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320319.429111:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1102 00000100:00100000:2.0:1767320319.429298:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983040128:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320319.430014:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040000 00000100:00100000:3.0:1767320319.430071:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103 00000100:00100000:3.0:1767320319.430080:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983040000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320319.430100:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320319.430153:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983040000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (1484us total) trans 0 rc 0/0 00000100:00080000:0.0:1767320319.430156:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320319.430163:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103 00000100:00100000:0.0:1767320319.430196:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983040000:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320320.132551:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320320.964660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320320.964672:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320320.964729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320320.964733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320320.964737:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320320.964744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5564 00000100:00080000:1.0:1767320320.964749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320320.964755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320320.964776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa680 x1853163983040384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320320.964810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320320.964813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320320.964815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320320.964819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5564 00000100:00080000:1.0:1767320320.964821:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320320.964825:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320320.964834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa300 x1853163983040640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320320.964843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320320.964846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320320.964847:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320320.964851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5564 00000100:00080000:1.0:1767320320.964854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320320.964857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320320.964867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9c00 x1853163983040768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320320.964882:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:1.0:1767320320.964897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320320.964900:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983040896:0@lo:400:kworker.0 00000100:00080000:1.0:1767320320.964902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320320.964906:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320320.964912:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5564 00000100:00080000:1.0:1767320320.964914:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320320.964917:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320320.964930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed4380 x1853163983041152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320320.964972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320320.964975:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983040256:0@lo:400:kworker.0 00000100:00100000:0.0:1767320320.965009:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983040512:0@lo:400:kworker.0 00000100:00100000:1.0:1767320320.965015:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320320.965036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320320.965060:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320320.965069:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983041024:0@lo:400:kworker.0 00000100:00100000:0.0:1767320320.965088:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040896 00000100:00100000:1.0:1767320320.965095:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320320.965096:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040896 00000100:00100000:1.0:1767320320.965134:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040256 00000100:00100000:1.0:1767320320.965139:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040256 00000100:00100000:0.0:1767320320.965146:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041024 00000100:00100000:0.0:1767320320.965150:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041024 00000100:00100000:0.0:1767320320.965185:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1767320320.965185:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040512 00000100:00100000:1.0:1767320320.965191:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040512 00000100:00100000:0.0:1767320320.965198:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983040896:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320320.965218:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1643 00000100:00100000:1.0:1767320320.965226:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983040256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320320.965251:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983040896:12345-0@lo:400:kworker.0 Request processed in 56us (280us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320320.965259:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023 00000100:00100000:0.0:1767320320.965301:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024 00000100:00100000:0.0:1767320320.965307:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983041024:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320320.965458:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1644 00000100:00100000:2.0:1767320320.965467:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983040512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320320.965721:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983040256:12345-0@lo:400:kworker.0 Request processed in 495us (705us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320320.965728:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1643 00000100:00100000:3.0:1767320320.965885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983040384:0@lo:38:kworker.0 00000100:00100000:0.0:1767320320.968829:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983041024:12345-0@lo:400:kworker.0 Request processed in 3512us (3724us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320320.968843:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024 00000100:00080000:0.0:1767320320.968897:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320320.968940:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983040896:0@lo:400:kworker.0 00010000:00080000:1.0:1767320320.969059:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0ab800 x1853163983040256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320336 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320320.969069:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983040256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320320.969186:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983040512:12345-0@lo:400:kworker.0 Request processed in 3718us (4148us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320320.969196:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1644 00000100:00100000:3.0:1767320320.969216:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320320.969243:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983040640:0@lo:38:kworker.0 00000100:00100000:2.0:1767320320.969258:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040384 00000100:00100000:3.0:1767320320.969264:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320320.969266:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040384 00000100:00100000:3.0:1767320320.969293:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983040768:0@lo:38:kworker.0 00000100:00100000:2.0:1767320320.969311:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040640 00000100:00100000:3.0:1767320320.969315:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320320.969317:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040640 00000100:00100000:3.0:1767320320.969332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983041152:0@lo:38:kworker.0 00000100:00100000:2.0:1767320320.969339:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172 00000100:00100000:2.0:1767320320.969346:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983040384:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320320.969351:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320320.969371:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320320.969383:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983040768 00000100:00100000:3.0:1767320320.969389:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983040768 00000100:00100000:2.0:1767320320.969416:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983040384:12345-0@lo:38:kworker.0 Request processed in 68us (201us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320320.969422:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 568 00000100:00100000:2.0:1767320320.969426:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172 00000100:00100000:3.0:1767320320.969431:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983040768:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320320.969450:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041152 00000100:00100000:2.0:1767320320.969454:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041152 00010000:00020000:3.0:1767320320.969456:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320320.969467:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2173 00000100:00100000:2.0:1767320320.969472:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983040640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320320.969482:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320320.969509:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983041024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320320.969550:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2174 00000100:00100000:1.0:1767320320.969555:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983041152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320320.969565:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:0.0:1767320320.969700:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9180 x1853163983040512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320336 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320320.969711:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983040512:0@lo:400:kworker.0 00000100:00100000:3.0:1767320320.970184:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983040768:12345-0@lo:38:kworker.0 Request processed in 751us (867us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320320.970187:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983040640:12345-0@lo:38:kworker.0 Request processed in 710us (919us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320320.970192:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983041152:12345-0@lo:38:kworker.0 Request processed in 633us (841us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320320.970196:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 568 00000100:00100000:2.0:1767320320.970205:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2173 00000100:00100000:1.0:1767320320.970211:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2174 00000100:00080000:3.0:1767320320.970231:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320320.970237:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320320.970241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983040384:0@lo:38:kworker.0 00000100:00080000:3.0:1767320320.970251:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320320.970254:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320320.970257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983040640:0@lo:38:kworker.0 00000100:00080000:3.0:1767320320.970265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320320.970268:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320320.970291:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983040768:0@lo:38:kworker.0 00000100:00080000:3.0:1767320320.970300:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320320.970302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320320.970305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983041152:0@lo:38:kworker.0 00010000:00010000:1.0:1767320321.158725:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320322.181875:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320322.960990:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320322.961096:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320322.965560:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674880 00000100:00100000:1.0:1767320322.965578:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674880 00000100:00100000:1.0:1767320322.965661:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915674752 00000100:00100000:1.0:1767320322.965664:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915674752 00000100:00100000:1.0:1767320322.965680:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1645 00000100:00100000:1.0:1767320322.965688:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915674880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320322.965760:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915674880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 73us (4783us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320322.965766:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1645 00000100:00100000:1.0:1767320322.965779:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2175 00000100:00100000:1.0:1767320322.965782:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915674752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320322.965798:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320322.965816:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915674752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 33us (4722us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320322.965820:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2175 00000100:00080000:0.0:1767320322.969023:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320323.204716:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320324.229921:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320324.548569:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983041280:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320324.548652:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320324.549103:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320324.549114:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983041408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320324.549158:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320324.550476:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041280 00000100:00100000:3.0:1767320324.550487:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041280 00000100:00100000:3.0:1767320324.550589:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041408 00000100:00100000:3.0:1767320324.550594:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041408 00000100:00100000:2.0:1767320324.550640:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104 00000100:00100000:2.0:1767320324.550652:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983041280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320324.550683:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320324.550745:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983041280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (2092us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320324.550752:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104 00000100:00100000:2.0:1767320324.550817:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983041280:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320324.551557:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320324.551563:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320324.551949:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1105 00000100:00100000:3.0:1767320324.551960:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983041408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320324.551986:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320324.552062:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983041408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (2902us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320324.552069:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1105 00000100:00100000:2.0:1767320324.552182:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983041408:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320325.253414:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320326.084493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320326.084503:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320326.084611:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983041536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320326.084685:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.084770:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041536 00000100:00100000:3.0:1767320326.084780:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041536 00000100:00100000:3.0:1767320326.084851:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1646 00000100:00100000:3.0:1767320326.084861:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983041536:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320326.084931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320326.084936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320326.084941:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320326.084946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5569 00000100:00080000:0.0:1767320326.084950:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320326.084955:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320326.084978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163983041664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320326.085016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320326.085019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320326.085021:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320326.085024:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5569 00000100:00080000:0.0:1767320326.085026:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320326.085030:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320326.085040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625880 x1853163983041920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320326.085049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320326.085051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320326.085053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320326.085056:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5569 00000100:00080000:0.0:1767320326.085058:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320326.085061:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320326.085072:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626300 x1853163983042048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320326.085161:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320326.085178:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983042304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320326.085217:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320326.085268:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983041792:0@lo:400:kworker.0 00000100:00100000:3.0:1767320326.085295:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983041536:12345-0@lo:400:kworker.0 Request processed in 413us (587us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.085305:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1646 00000100:00100000:1.0:1767320326.085320:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320326.085352:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320326.085360:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983042176:0@lo:400:kworker.0 00000100:00080000:0.0:1767320326.085380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320326.085386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320326.085388:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320326.085390:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320326.085397:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5569 00000100:00080000:0.0:1767320326.085400:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320326.085404:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320326.085419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627800 x1853163983042432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320326.085467:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042304 00000100:00100000:0.0:1767320326.085472:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042304 00000100:00100000:0.0:1767320326.085503:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042176 00000100:00100000:0.0:1767320326.085506:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042176 00000100:00100000:0.0:1767320326.085528:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025 00000100:00100000:0.0:1767320326.085535:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983042304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320326.085583:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983042304:12345-0@lo:400:kworker.0 Request processed in 48us (365us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320326.085589:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025 00000100:00100000:0.0:1767320326.085601:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026 00000100:00100000:0.0:1767320326.085607:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983042176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320326.085639:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983042176:12345-0@lo:400:kworker.0 Request processed in 32us (253us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320326.085645:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026 00000100:00100000:0.0:1767320326.085682:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983041664:0@lo:38:kworker.0 00000100:00100000:0.0:1767320326.085706:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767320326.086130:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c627b80 x1853163983041536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320342 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320326.086145:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983041536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320326.091777:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983041920:0@lo:38:kworker.0 00000100:00100000:0.0:1767320326.091855:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320326.094760:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041792 00000100:00100000:1.0:1767320326.094761:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041664 00000100:00100000:1.0:1767320326.094777:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041664 00000100:00100000:3.0:1767320326.094778:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041792 00000100:00100000:1.0:1767320326.094828:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983041920 00000100:00100000:1.0:1767320326.094832:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983041920 00000100:00100000:3.0:1767320326.094837:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1647 00000100:00100000:0.0:1767320326.094848:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983042048:0@lo:38:kworker.0 00000100:00100000:3.0:1767320326.094848:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983041792:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320326.094854:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2176 00000100:00100000:1.0:1767320326.094861:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983041664:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320326.094880:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320326.095686:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2177 00000100:00100000:2.0:1767320326.095693:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983041920:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320326.095707:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320326.096445:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983041792:12345-0@lo:400:kworker.0 Request processed in 1593us (11119us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320326.096456:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1647 00000100:00100000:2.0:1767320326.096491:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983041920:12345-0@lo:38:kworker.0 Request processed in 786us (4624us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320326.096501:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2177 00000100:00100000:2.0:1767320326.096579:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983042304:0@lo:400:kworker.0 00000100:00100000:2.0:1767320326.096612:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983042176:0@lo:400:kworker.0 00010000:00080000:2.0:1767320326.096625:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626d80 x1853163983041792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320342 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320326.096635:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983041792:0@lo:400:kworker.0 00000100:00080000:2.0:1767320326.096679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320326.096685:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320326.096688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320326.096891:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983041664:12345-0@lo:38:kworker.0 Request processed in 2027us (11182us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320326.096899:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2176 00000100:00100000:0.0:1767320326.096947:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320326.097031:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983042432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320326.097063:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320326.097105:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320326.097110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320326.097114:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983041664:0@lo:38:kworker.0 00000100:00080000:0.0:1767320326.097137:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320326.097141:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320326.097145:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983041920:0@lo:38:kworker.0 00000100:00100000:1.0:1767320326.101447:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042432 00000100:00100000:1.0:1767320326.101468:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042432 00000100:00100000:1.0:1767320326.101542:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178 00000100:00100000:1.0:1767320326.101552:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983042432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320326.101575:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320326.101666:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983042432:12345-0@lo:38:kworker.0 Request processed in 112us (4600us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320326.101678:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178 00000100:00080000:0.0:1767320326.101701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320326.101708:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320326.101713:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983042432:0@lo:38:kworker.0 00000100:00100000:2.0:1767320326.102258:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042048 00000100:00100000:2.0:1767320326.102264:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042048 00000100:00100000:2.0:1767320326.102314:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 569 00000100:00100000:2.0:1767320326.102320:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983042048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320326.102332:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320326.102384:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983042048:12345-0@lo:38:kworker.0 Request processed in 62us (5436us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320326.102390:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 569 00000100:00080000:0.0:1767320326.102594:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320326.102600:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320326.102603:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983042048:0@lo:38:kworker.0 00010000:00010000:2.0:1767320326.276599:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320327.302249:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320328.077097:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320328.077161:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320328.077339:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675008 00000100:00100000:0.0:1767320328.077354:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675008 00000100:00100000:2.0:1767320328.077356:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675136 00000100:00100000:2.0:1767320328.077371:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675136 00000100:00100000:0.0:1767320328.077423:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2179 00000100:00100000:0.0:1767320328.077435:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915675008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320328.077465:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320328.077478:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1648 00000100:00100000:2.0:1767320328.077493:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915675136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320328.077554:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915675008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 118us (480us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320328.077564:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2179 00000100:00100000:2.0:1767320328.077578:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915675136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (413us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320328.077589:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1648 00000100:00080000:3.0:1767320328.080264:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320328.326238:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320329.352155:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320329.668597:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320329.668629:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983042560:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320329.668714:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320329.668928:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042560 00000100:00100000:1.0:1767320329.668938:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042560 00000100:00100000:1.0:1767320329.669013:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106 00000100:00100000:1.0:1767320329.669024:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983042560:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320329.669063:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320329.669121:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983042560:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (406us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320329.669129:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106 00000100:00080000:1.0:1767320329.669154:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320329.669209:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983042560:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320329.669488:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983042688:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320329.669522:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320329.669634:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042688 00000100:00100000:3.0:1767320329.669640:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042688 00000100:00100000:3.0:1767320329.669693:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107 00000100:00100000:3.0:1767320329.669701:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983042688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320329.669732:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320329.669772:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983042688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (249us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320329.669778:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107 00000100:00100000:3.0:1767320329.669807:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983042688:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320329.672751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320330.372547:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320331.204566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320331.204580:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320331.204667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320331.204672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320331.204677:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320331.204688:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5574 00000100:00080000:1.0:1767320331.204694:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320331.204703:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320331.204739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab800 x1853163983042944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320331.204798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320331.204801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320331.204803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320331.204808:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5574 00000100:00080000:1.0:1767320331.204811:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320331.204814:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320331.204829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57b80 x1853163983043200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320331.204837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320331.204840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320331.204842:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320331.204846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5574 00000100:00080000:1.0:1767320331.204848:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320331.204852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320331.204867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594000 x1853163983043328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320331.204888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320331.204892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320331.204894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320331.204897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5574 00000100:00080000:1.0:1767320331.204900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320331.204903:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320331.204914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163983043712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320331.204944:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983043072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320331.205034:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320331.205111:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320331.205117:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983043456:0@lo:400:kworker.0 00000100:00100000:1.0:1767320331.205148:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320331.205182:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043072 00000100:00100000:1.0:1767320331.205191:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043072 00000100:00100000:1.0:1767320331.205267:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1649 00000100:00100000:1.0:1767320331.205313:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983043072:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320331.205380:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983043072:12345-0@lo:400:kworker.0 Request processed in 70us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320331.205389:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1649 00010000:00080000:1.0:1767320331.205437:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aa680 x1853163983043072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320347 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320331.205451:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983043072:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.205642:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983043584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.205693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320331.205729:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320331.205736:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983042816:0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.205760:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320331.205803:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320331.205818:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042816 00000100:00100000:2.0:1767320331.205822:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042816 00000100:00100000:2.0:1767320331.205859:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1650 00000100:00100000:2.0:1767320331.205866:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983042816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320331.205903:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983042816:12345-0@lo:400:kworker.0 Request processed in 39us (142us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320331.205909:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1650 00000100:00080000:2.0:1767320331.205928:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320331.205945:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9180 x1853163983042816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320347 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320331.205955:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983042816:0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.206389:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043456 00000100:00100000:0.0:1767320331.206395:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043456 00000100:00100000:0.0:1767320331.206447:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043584 00000100:00100000:0.0:1767320331.206450:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043584 00000100:00100000:0.0:1767320331.206473:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1027 00000100:00100000:0.0:1767320331.206481:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983043456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.206534:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983043456:12345-0@lo:400:kworker.0 Request processed in 54us (1385us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320331.206541:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1027 00000100:00100000:0.0:1767320331.206555:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767320331.206560:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983043584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320331.206593:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983043584:12345-0@lo:400:kworker.0 Request processed in 32us (900us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320331.206598:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028 00000100:00100000:0.0:1767320331.206636:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983042944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.206670:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.206701:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983043200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.206738:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.206756:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983043328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.206864:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.206892:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983043712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.206916:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320331.206953:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983042944 00000100:00100000:0.0:1767320331.206957:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983042944 00000100:00100000:0.0:1767320331.206977:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043200 00000100:00100000:0.0:1767320331.206980:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043200 00000100:00100000:0.0:1767320331.206992:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043712 00000100:00100000:0.0:1767320331.206994:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043712 00000100:00100000:0.0:1767320331.207004:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2180 00000100:00100000:0.0:1767320331.207009:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983042944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320331.207032:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320331.207071:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983042944:12345-0@lo:38:kworker.0 Request processed in 60us (400us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320331.207077:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2180 00000100:00100000:0.0:1767320331.207088:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181 00000100:00100000:0.0:1767320331.207092:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983043200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320331.207100:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320331.207126:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983043200:12345-0@lo:38:kworker.0 Request processed in 32us (387us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320331.207131:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181 00000100:00100000:0.0:1767320331.207142:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2182 00000100:00100000:0.0:1767320331.207145:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983043712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320331.207153:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320331.207175:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983043712:12345-0@lo:38:kworker.0 Request processed in 28us (258us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320331.207180:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2182 00000100:00100000:0.0:1767320331.207198:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043328 00000100:00100000:0.0:1767320331.207202:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043328 00000100:00100000:0.0:1767320331.207219:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 570 00000100:00100000:0.0:1767320331.207224:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983043328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320331.207233:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320331.207303:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983043328:12345-0@lo:38:kworker.0 Request processed in 45us (406us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320331.207309:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 570 00000100:00100000:0.0:1767320331.207346:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983043584:0@lo:400:kworker.0 00000100:00080000:0.0:1767320331.207377:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320331.207382:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320331.207385:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983042944:0@lo:38:kworker.0 00000100:00080000:0.0:1767320331.207394:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320331.207397:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320331.207400:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983043200:0@lo:38:kworker.0 00000100:00080000:0.0:1767320331.207408:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320331.207411:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320331.207414:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983043328:0@lo:38:kworker.0 00000100:00080000:0.0:1767320331.207424:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320331.207427:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320331.207429:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983043712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320331.207467:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983043456:0@lo:400:kworker.0 00000100:00080000:2.0:1767320331.208142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320331.208150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320331.396694:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320332.420583:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320333.195982:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320333.196244:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675392 00000100:00100000:2.0:1767320333.196255:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675392 00000100:00100000:2.0:1767320333.196355:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1651 00000100:00100000:2.0:1767320333.196366:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915675392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320333.196454:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915675392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (491us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320333.196461:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1651 00000100:00080000:0.0:1767320333.197304:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320333.197816:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320333.198257:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675264 00000100:00100000:3.0:1767320333.198266:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675264 00000100:00100000:3.0:1767320333.198329:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2183 00000100:00100000:3.0:1767320333.198336:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915675264:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320333.198359:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320333.198417:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915675264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 80us (603us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320333.198424:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2183 00010000:00010000:1.0:1767320333.445134:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320334.468431:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320334.788641:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320334.788661:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983043840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320334.788734:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320334.788851:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043840 00000100:00100000:1.0:1767320334.788861:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043840 00000100:00100000:1.0:1767320334.788930:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108 00000100:00100000:1.0:1767320334.788949:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983043840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320334.788976:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320334.789021:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983043840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320334.789028:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108 00000100:00080000:0.0:1767320334.789090:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320334.789135:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983043840:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320334.795876:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320334.795897:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983043968:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320334.795985:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320334.796136:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983043968 00000100:00100000:1.0:1767320334.796144:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983043968 00000100:00100000:1.0:1767320334.796214:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109 00000100:00100000:1.0:1767320334.796224:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983043968:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320334.796254:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320334.796318:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983043968:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320334.796326:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109 00000100:00100000:1.0:1767320334.796371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983043968:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320334.796672:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320335.492466:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320336.324659:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320336.324669:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320336.324729:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320336.324741:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320336.324745:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320336.324752:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5579 00000100:00080000:2.0:1767320336.324757:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320336.324763:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320336.324785:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582a00 x1853163983044224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320336.324834:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320336.324837:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320336.324839:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320336.324843:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5579 00000100:00080000:2.0:1767320336.324845:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320336.324849:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320336.324861:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90a80 x1853163983044480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320336.324870:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320336.324873:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320336.324874:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320336.324878:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5579 00000100:00080000:2.0:1767320336.324880:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320336.324883:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320336.324893:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91180 x1853163983044608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320336.324911:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320336.324914:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320336.324915:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320336.324919:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5579 00000100:00080000:2.0:1767320336.324921:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320336.324924:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320336.324933:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163983044992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320336.324960:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983044736:0@lo:400:kworker.0 00000100:00100000:2.0:1767320336.325019:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320336.325074:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320336.325079:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983044352:0@lo:400:kworker.0 00000100:00100000:2.0:1767320336.325103:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320336.325145:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044352 00000100:00100000:2.0:1767320336.325180:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044352 00000100:00100000:0.0:1767320336.325221:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983044864:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.325361:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320336.325388:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320336.325395:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983044096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.325418:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.325451:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983044224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320336.325455:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044736 00000100:00100000:0.0:1767320336.325461:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044736 00000100:00100000:3.0:1767320336.325486:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.325520:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983044480:0@lo:38:kworker.0 00000100:00100000:3.0:1767320336.325539:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.325552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983044608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320336.325569:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.325585:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983044992:0@lo:38:kworker.0 00000100:00100000:3.0:1767320336.325600:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320336.325626:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044096 00000100:00100000:3.0:1767320336.325630:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044096 00000100:00100000:3.0:1767320336.325648:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044224 00000100:00100000:3.0:1767320336.325650:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044224 00000100:00100000:3.0:1767320336.325656:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044480 00000100:00100000:3.0:1767320336.325658:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044480 00000100:00100000:3.0:1767320336.325664:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044992 00000100:00100000:3.0:1767320336.325666:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044992 00000100:00100000:3.0:1767320336.325675:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1767320336.325683:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983044352:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320336.325725:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983044352:12345-0@lo:400:kworker.0 Request processed in 43us (621us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320336.325744:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1767320336.325760:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1653 00000100:00100000:3.0:1767320336.325766:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983044096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320336.325808:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983044096:12345-0@lo:400:kworker.0 Request processed in 43us (389us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320336.325817:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1653 00000100:00100000:3.0:1767320336.325827:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184 00000100:00100000:3.0:1767320336.325831:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320336.325846:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320336.325870:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044224:12345-0@lo:38:kworker.0 Request processed in 38us (385us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320336.325875:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184 00000100:00100000:3.0:1767320336.325884:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2185 00000100:00100000:3.0:1767320336.325887:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044480:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320336.325894:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320336.325911:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044480:12345-0@lo:38:kworker.0 Request processed in 22us (371us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320336.325916:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2185 00000100:00100000:3.0:1767320336.325925:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2186 00000100:00100000:3.0:1767320336.325928:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044992:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320336.325935:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320336.325953:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983044992:12345-0@lo:38:kworker.0 Request processed in 23us (352us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320336.325958:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2186 00000100:00080000:3.0:1767320336.325970:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320336.325974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320336.325993:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320336.325997:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320336.326000:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983044224:0@lo:38:kworker.0 00000100:00080000:3.0:1767320336.326009:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320336.326011:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320336.326014:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983044480:0@lo:38:kworker.0 00000100:00080000:3.0:1767320336.326019:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320336.326021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320336.326024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983044992:0@lo:38:kworker.0 00000100:00100000:3.0:1767320336.326118:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044608 00000100:00100000:3.0:1767320336.326122:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044608 00000100:00100000:3.0:1767320336.326141:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 571 00000100:00100000:3.0:1767320336.326146:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983044608:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320336.326168:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320336.326200:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983044608:12345-0@lo:38:kworker.0 Request processed in 54us (631us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320336.326206:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 571 00000100:00080000:3.0:1767320336.326239:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320336.326243:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320336.326246:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983044608:0@lo:38:kworker.0 00010000:00080000:3.0:1767320336.326305:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582680 x1853163983044352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320352 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320336.326315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983044352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.326354:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983044864 00000100:00100000:0.0:1767320336.326359:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983044864 00010000:00080000:3.0:1767320336.326381:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581500 x1853163983044096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320352 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320336.326391:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983044096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.326396:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029 00000100:00100000:0.0:1767320336.326409:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983044736:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320336.326422:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320336.326436:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1030 00000100:00100000:3.0:1767320336.326445:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983044864:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320336.326454:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983044736:12345-0@lo:400:kworker.0 Request processed in 46us (1433us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320336.326462:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029 00000100:00100000:3.0:1767320336.326488:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983044864:12345-0@lo:400:kworker.0 Request processed in 43us (1231us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320336.326495:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1030 00000100:00100000:0.0:1767320336.326499:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983044736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320336.326527:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983044864:0@lo:400:kworker.0 00010000:00010000:1.0:1767320336.516609:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320337.540678:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320338.321744:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320338.321843:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915675520 offset 0 length 520: 4 00000100:00100000:1.0:1767320338.321868:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320338.322019:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675776 00000100:00100000:2.0:1767320338.322041:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675776 00000100:00100000:2.0:1767320338.322130:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675648 00000100:00100000:2.0:1767320338.322134:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675648 00000100:00100000:2.0:1767320338.322157:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1654 00000100:00100000:2.0:1767320338.322167:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915675776:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320338.322248:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915675776:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (513us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320338.322256:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1654 00000100:00100000:1.0:1767320338.322390:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2187 00000100:00100000:1.0:1767320338.322395:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915675648:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320338.322416:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320338.322446:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915675648:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (577us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320338.322453:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2187 00000100:00080000:0.0:1767320338.322984:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320338.564478:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320339.589666:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320339.908606:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320339.908629:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983045120:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320339.908716:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320339.908924:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045120 00000100:00100000:2.0:1767320339.908933:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045120 00000100:00100000:2.0:1767320339.909006:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110 00000100:00100000:2.0:1767320339.909015:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983045120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320339.909043:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320339.909100:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983045120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (383us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320339.909109:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110 00000100:00100000:2.0:1767320339.909162:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983045120:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320339.909299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320339.909428:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983045248:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320339.909460:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320339.909574:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045248 00000100:00100000:1.0:1767320339.909579:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045248 00000100:00100000:1.0:1767320339.909621:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1111 00000100:00100000:1.0:1767320339.909629:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983045248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320339.909646:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320339.909760:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320339.909827:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983045248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 198us (365us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320339.909849:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1111 00000100:00100000:3.0:1767320339.913012:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983045248:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320340.420408:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320315/real 1767320315] req@ffff93463e6f0380 x1853163983038848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320340 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320340.420444:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463e6f0380 x1853163983038848/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320340 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320340.420464:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320340.420514:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320340.420518:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983038848:0@lo:250:kworker.0 00000100:00080000:1.0:1767320340.423134:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320340.423144:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320340.423147:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320340.423155:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5564 00000100:00080000:1.0:1767320340.423160:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320340.423168:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320340.423206:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af1500 x1853163983045376/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320340.423307:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983045504:0@lo:400:kworker.0 00000100:00100000:0.0:1767320340.423374:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320340.423492:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045504 00000100:00100000:2.0:1767320340.423499:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045504 00000100:00100000:2.0:1767320340.423555:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1655 00000100:00100000:2.0:1767320340.423565:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983045504:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320340.423653:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983045376:0@lo:250:kworker.0 00000400:00000100:3.0:1767320340.423675:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983045376 offset 0 length 520: 4 00000100:00080000:3.0:1767320340.423721:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320340.426323:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320340.426334:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320340.426337:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320340.426346:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5584 00000100:00080000:1.0:1767320340.426351:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320340.426357:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320340.426396:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab800 x1853163983045632/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320340.426440:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320340.426443:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320340.426445:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320340.426448:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5584 00000100:00080000:1.0:1767320340.426450:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320340.426454:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320340.426465:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa680 x1853163983045888/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320340.426473:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320340.426476:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320340.426478:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320340.426481:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5584 00000100:00080000:1.0:1767320340.426483:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320340.426487:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320340.426502:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b100 x1853163983046016/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320340.426512:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320340.426515:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320340.426517:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320340.426520:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5584 00000100:00080000:1.0:1767320340.426522:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320340.426525:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320340.426533:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163983046144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320340.426578:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320340.426595:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983045760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320340.426656:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320340.426928:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983045632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320340.426959:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320340.426987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983045888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320340.427007:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320340.427028:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983046016:0@lo:38:kworker.0 00000100:00100000:3.0:1767320340.427061:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320340.427088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983046144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320340.427109:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320340.427151:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046016 00000100:00100000:3.0:1767320340.427158:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046016 00000100:00100000:3.0:1767320340.427191:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 572 00000100:00100000:3.0:1767320340.427198:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983046016:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320340.427217:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320340.427258:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983046016:12345-0@lo:38:kworker.0 Request processed in 60us (195us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320340.427266:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 572 00000100:00100000:3.0:1767320340.427310:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045760 00000100:00100000:3.0:1767320340.427314:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045760 00000100:00100000:3.0:1767320340.427347:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045632 00000100:00100000:3.0:1767320340.427350:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045632 00000100:00100000:3.0:1767320340.427358:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983045888 00000100:00100000:3.0:1767320340.427359:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983045888 00000100:00100000:3.0:1767320340.427365:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046144 00000100:00100000:3.0:1767320340.427366:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046144 00000100:00100000:3.0:1767320340.427375:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656 00000100:00100000:3.0:1767320340.427380:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983045760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320340.427422:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983045760:12345-0@lo:400:kworker.0 Request processed in 41us (765us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320340.427428:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656 00000100:00100000:3.0:1767320340.427440:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2188 00000100:00100000:3.0:1767320340.427444:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983045632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320340.427454:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320340.427476:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983045632:12345-0@lo:38:kworker.0 Request processed in 31us (517us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320340.427482:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2188 00000100:00100000:3.0:1767320340.427493:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2189 00000100:00100000:3.0:1767320340.427496:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983045888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320340.427504:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320340.427524:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983045888:12345-0@lo:38:kworker.0 Request processed in 26us (516us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320340.427530:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2189 00000100:00100000:3.0:1767320340.427538:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190 00000100:00100000:3.0:1767320340.427541:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983046144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320340.427550:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320340.427571:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983046144:12345-0@lo:38:kworker.0 Request processed in 28us (461us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320340.427577:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190 00000100:00080000:3.0:1767320340.427593:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320340.427623:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9180 x1853163983045760/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320356 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320340.427636:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983045760:0@lo:400:kworker.0 00000100:00080000:3.0:1767320340.427661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320340.427666:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320340.427669:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983045632:0@lo:38:kworker.0 00000100:00080000:3.0:1767320340.427678:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320340.427680:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320340.427683:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983045888:0@lo:38:kworker.0 00000100:00080000:3.0:1767320340.427690:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320340.427693:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320340.427696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983046016:0@lo:38:kworker.0 00000100:00080000:3.0:1767320340.427704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320340.427707:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320340.427709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983046144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320340.431936:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983045504:12345-0@lo:400:kworker.0 Request processed in 8363us (8550us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320340.431960:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1655 00010000:00080000:0.0:1767320340.436940:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2d80 x1853163983045504/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320356 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320340.437051:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983045504:0@lo:400:kworker.0 00010000:00010000:2.0:1767320340.614023:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320341.636519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320342.661628:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320343.439553:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320343.439972:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676032 00000100:00100000:2.0:1767320343.439983:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676032 00000100:00100000:2.0:1767320343.440072:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1657 00000100:00100000:2.0:1767320343.440084:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320343.440172:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (628us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320343.440180:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1657 00000100:00100000:0.0:1767320343.440244:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320343.440341:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915675904 00000100:00100000:2.0:1767320343.440345:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915675904 00000100:00100000:2.0:1767320343.440366:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2191 00000100:00100000:2.0:1767320343.440370:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915675904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320343.440389:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320343.440413:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915675904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 41us (169us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320343.440418:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2191 00000100:00080000:3.0:1767320343.441047:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320343.687570:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320344.708445:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320345.028917:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983046400:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320345.028933:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320345.028953:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983046272:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320345.029033:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320345.029121:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046400 00000100:00100000:0.0:1767320345.029130:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046400 00000100:00100000:0.0:1767320345.029209:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112 00000100:00100000:0.0:1767320345.029219:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983046400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320345.029259:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320345.029322:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983046400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320345.029329:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112 00000100:00100000:0.0:1767320345.029371:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983046400:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320345.029728:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320345.029782:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320345.030019:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046272 00000100:00100000:0.0:1767320345.030025:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046272 00000100:00080000:3.0:1767320345.031181:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320345.031348:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113 00000100:00100000:0.0:1767320345.031362:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983046272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320345.031396:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320345.031478:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983046272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 121us (1696us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320345.031485:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113 00000100:00100000:3.0:1767320345.031568:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983046272:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320345.540491:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320345.540501:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320345.540564:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320345.540568:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320345.540572:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320345.540579:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5588 00000100:00080000:1.0:1767320345.540584:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320345.540591:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320345.540605:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320345.540616:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b100 x1853163983046656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320345.540622:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983046528:0@lo:400:kworker.0 00000100:00080000:1.0:1767320345.540674:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320345.540678:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320345.540680:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320345.540685:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5588 00000100:00080000:1.0:1767320345.540688:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320345.540692:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320345.540702:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320345.540707:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2300 x1853163983046912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320345.540719:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320345.540722:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320345.540725:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320345.540729:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5588 00000100:00080000:1.0:1767320345.540732:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320345.540736:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320345.540748:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163983047040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320345.540757:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320345.540770:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983047168:0@lo:400:kworker.0 00000100:00080000:1.0:1767320345.540780:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320345.540786:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320345.540788:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320345.540792:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5588 00000100:00080000:1.0:1767320345.540795:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320345.540799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320345.540807:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320345.540813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2d80 x1853163983047424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320345.540828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983046784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320345.540844:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046528 00000100:00100000:1.0:1767320345.540853:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046528 00000100:00100000:2.0:1767320345.540856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320345.540891:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046784 00000100:00100000:2.0:1767320345.540895:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046784 00000100:00100000:3.0:1767320345.540959:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983046656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320345.540967:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1658 00000100:00100000:2.0:1767320345.540977:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983046528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320345.541020:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320345.541047:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983046912:0@lo:38:kworker.0 00000100:00100000:3.0:1767320345.541069:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320345.541080:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983047040:0@lo:38:kworker.0 00000100:00100000:3.0:1767320345.541099:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320345.541115:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983047424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320345.541132:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983046528:12345-0@lo:400:kworker.0 Request processed in 156us (429us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320345.541137:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320345.541147:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1658 00000100:00100000:2.0:1767320345.541169:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046656 00000100:00100000:3.0:1767320345.541172:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983046912 00000100:00100000:2.0:1767320345.541173:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046656 00000100:00100000:3.0:1767320345.541180:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983046912 00000100:00100000:2.0:1767320345.541188:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047424 00000100:00100000:2.0:1767320345.541192:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047424 00000100:00100000:3.0:1767320345.541206:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1659 00000100:00100000:2.0:1767320345.541244:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2192 00000100:00100000:2.0:1767320345.541249:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983046656:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320345.541266:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320345.541362:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320345.541374:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983047296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320345.541404:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320345.541435:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047168 00000100:00100000:0.0:1767320345.541440:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047168 00000100:00100000:0.0:1767320345.541466:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047296 00000100:00100000:0.0:1767320345.541469:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047296 00000100:00100000:0.0:1767320345.541486:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031 00000100:00100000:0.0:1767320345.541492:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983047168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320345.541528:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983047168:12345-0@lo:400:kworker.0 Request processed in 36us (722us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320345.541535:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031 00000100:00100000:0.0:1767320345.541547:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032 00000100:00100000:0.0:1767320345.541552:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983047296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320345.541581:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983047296:12345-0@lo:400:kworker.0 Request processed in 29us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320345.541587:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032 00000100:00100000:0.0:1767320345.541626:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983047168:0@lo:400:kworker.0 00000100:00100000:0.0:1767320345.541649:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983047296:0@lo:400:kworker.0 00000100:00100000:2.0:1767320345.541685:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983046656:12345-0@lo:38:kworker.0 Request processed in 82us (313us total) trans 0 rc -19/-19 00010000:00080000:0.0:1767320345.541688:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435bb80 x1853163983046528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320361 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320345.541697:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2192 00000100:00100000:0.0:1767320345.541706:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983046528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320345.541717:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193 00000100:00100000:2.0:1767320345.541722:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983046912:12345-0@lo:38:kworker.0 00000100:00080000:0.0:1767320345.541732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:2.0:1767320345.541733:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320345.541762:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2194 00000100:00100000:2.0:1767320345.541767:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983046912:12345-0@lo:38:kworker.0 Request processed in 43us (696us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320345.541771:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983047424:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320345.541778:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193 00010000:00020000:0.0:1767320345.541787:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320345.541819:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983047424:12345-0@lo:38:kworker.0 Request processed in 49us (684us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320345.541825:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2194 00000100:00100000:2.0:1767320345.541904:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047040 00000100:00100000:2.0:1767320345.541908:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047040 00000100:00100000:2.0:1767320345.541930:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 573 00000100:00100000:2.0:1767320345.541935:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983047040:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320345.541946:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320345.541956:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320345.541963:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320345.541967:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983046656:0@lo:38:kworker.0 00000100:00100000:2.0:1767320345.541979:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983047040:12345-0@lo:38:kworker.0 Request processed in 41us (877us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320345.541981:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320345.541999:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320345.542003:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983046912:0@lo:38:kworker.0 00000100:00080000:0.0:1767320345.542011:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320345.542014:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320345.542017:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983047040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320345.542023:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320345.542026:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320345.542028:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983047424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320345.542115:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983046784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320345.542156:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983046784:12345-0@lo:400:kworker.0 Request processed in 943us (1302us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320345.542163:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1659 00010000:00080000:0.0:1767320345.542734:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3100 x1853163983046784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320361 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320345.542746:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983046784:0@lo:400:kworker.0 00000100:00100000:2.0:1767320345.542959:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 573 00010000:00010000:1.0:1767320345.732654:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320346.756566:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320347.784188:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320348.557779:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320348.557906:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320348.558301:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676160 00000100:00100000:2.0:1767320348.558318:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676160 00000100:00100000:2.0:1767320348.558365:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676288 00000100:00100000:2.0:1767320348.558367:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676288 00000100:00100000:2.0:1767320348.558440:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1660 00000100:00100000:2.0:1767320348.558447:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915676288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320348.558506:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915676288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 60us (602us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320348.558511:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1660 00000100:00100000:2.0:1767320348.558523:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2195 00000100:00100000:2.0:1767320348.558526:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915676160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320348.558542:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320348.558557:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915676160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 29us (792us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320348.558560:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2195 00000100:00080000:1.0:1767320348.559690:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320348.806079:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320349.828573:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320350.148774:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320350.148799:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983047552:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320350.148872:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320350.149153:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983047680:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320350.149185:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320350.149424:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047552 00000100:00100000:1.0:1767320350.149434:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047552 00000100:00100000:1.0:1767320350.149494:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047680 00000100:00100000:1.0:1767320350.149498:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047680 00000100:00100000:1.0:1767320350.149518:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1114 00000100:00100000:1.0:1767320350.149527:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983047552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320350.149556:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320350.149612:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983047552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (739us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320350.149619:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1114 00000100:00100000:1.0:1767320350.149637:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115 00000100:00100000:1.0:1767320350.149644:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983047680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320350.149658:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320350.149689:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983047680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320350.149696:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115 00000100:00100000:1.0:1767320350.149739:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983047680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320350.149770:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983047552:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320350.149915:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320350.149920:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320350.667375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320350.667386:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320350.667464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320350.667468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320350.667472:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320350.667481:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5594 00000100:00080000:3.0:1767320350.667486:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320350.667493:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320350.667519:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0700 x1853163983047936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320350.667587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320350.667591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320350.667593:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320350.667597:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5594 00000100:00080000:3.0:1767320350.667600:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320350.667604:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320350.667616:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163983048192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320350.667626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320350.667629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320350.667630:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320350.667634:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5594 00000100:00080000:3.0:1767320350.667636:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320350.667640:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320350.667651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1880 x1853163983048320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320350.667671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320350.667674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320350.667676:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320350.667679:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5594 00000100:00080000:3.0:1767320350.667681:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320350.667683:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320350.667717:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb800 x1853163983048704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320350.667754:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983048448:0@lo:400:kworker.0 00000100:00100000:3.0:1767320350.667821:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320350.667873:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320350.667879:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983048064:0@lo:400:kworker.0 00000100:00100000:3.0:1767320350.667940:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320350.667972:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048064 00000100:00100000:3.0:1767320350.667978:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048064 00000100:00100000:3.0:1767320350.668032:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1661 00000100:00100000:3.0:1767320350.668042:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983048064:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320350.668092:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983048064:12345-0@lo:400:kworker.0 Request processed in 52us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320350.668099:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1661 00000100:00080000:3.0:1767320350.668121:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320350.668147:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1c00 x1853163983048064/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320366 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320350.668160:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983048064:0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668256:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983047808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668308:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320350.668335:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320350.668341:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983048576:0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668363:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320350.668390:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048448 00000100:00100000:0.0:1767320350.668395:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048448 00000100:00100000:0.0:1767320350.668425:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048576 00000100:00100000:0.0:1767320350.668428:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048576 00000100:00100000:0.0:1767320350.668448:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1033 00000100:00100000:0.0:1767320350.668455:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983048448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668492:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983048448:12345-0@lo:400:kworker.0 Request processed in 38us (671us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320350.668499:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1033 00000100:00100000:0.0:1767320350.668512:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034 00000100:00100000:0.0:1767320350.668518:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983048576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668551:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983048576:12345-0@lo:400:kworker.0 Request processed in 32us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320350.668556:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034 00000100:00100000:0.0:1767320350.668573:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047808 00000100:00100000:0.0:1767320350.668577:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047808 00000100:00100000:0.0:1767320350.668606:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662 00000100:00100000:0.0:1767320350.668613:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983047808:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668643:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983047808:12345-0@lo:400:kworker.0 Request processed in 31us (334us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320350.668650:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662 00000100:00080000:0.0:1767320350.668691:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320350.668711:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983048576:0@lo:400:kworker.0 00010000:00080000:0.0:1767320350.668722:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582a00 x1853163983047808/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320366 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320350.668732:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983047808:0@lo:400:kworker.0 00000100:00100000:0.0:1767320350.668758:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983048448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320350.669671:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983047936:0@lo:38:kworker.0 00000100:00100000:2.0:1767320350.669730:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320350.669773:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983048192:0@lo:38:kworker.0 00000100:00100000:2.0:1767320350.669797:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320350.669835:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983048320:0@lo:38:kworker.0 00000100:00100000:2.0:1767320350.669861:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320350.669917:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983048704:0@lo:38:kworker.0 00000100:00100000:2.0:1767320350.669940:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320350.669977:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048320 00000100:00100000:2.0:1767320350.669984:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048320 00000100:00100000:2.0:1767320350.670014:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 574 00000100:00100000:2.0:1767320350.670021:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983048320:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320350.670042:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320350.670080:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983048320:12345-0@lo:38:kworker.0 Request processed in 58us (217us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320350.670087:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 574 00000100:00100000:2.0:1767320350.670109:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983047936 00000100:00100000:2.0:1767320350.670113:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983047936 00000100:00100000:2.0:1767320350.670129:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048192 00000100:00100000:2.0:1767320350.670131:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048192 00000100:00100000:2.0:1767320350.670138:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048704 00000100:00100000:2.0:1767320350.670140:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048704 00000100:00100000:2.0:1767320350.670148:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196 00000100:00100000:2.0:1767320350.670152:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983047936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320350.670161:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320350.670186:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983047936:12345-0@lo:38:kworker.0 Request processed in 32us (456us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320350.670192:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196 00000100:00100000:2.0:1767320350.670202:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2197 00000100:00100000:2.0:1767320350.670205:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983048192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320350.670214:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320350.670234:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983048192:12345-0@lo:38:kworker.0 Request processed in 27us (436us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320350.670240:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2197 00000100:00100000:2.0:1767320350.670250:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2198 00000100:00100000:2.0:1767320350.670253:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983048704:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320350.670262:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320350.670311:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983048704:12345-0@lo:38:kworker.0 Request processed in 56us (370us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320350.670317:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2198 00000100:00080000:2.0:1767320350.670351:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320350.670356:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320350.670360:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983047936:0@lo:38:kworker.0 00000100:00080000:2.0:1767320350.670369:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320350.670372:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320350.670374:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983048192:0@lo:38:kworker.0 00000100:00080000:2.0:1767320350.670380:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320350.670383:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320350.670386:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983048320:0@lo:38:kworker.0 00000100:00080000:2.0:1767320350.670391:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320350.670394:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320350.670397:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983048704:0@lo:38:kworker.0 00010000:00010000:2.0:1767320350.856378:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320351.877801:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320352.900900:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320353.675775:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320353.679871:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676544 00000100:00100000:3.0:1767320353.679899:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676544 00000100:00100000:3.0:1767320353.680031:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1663 00000100:00100000:3.0:1767320353.680041:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767320353.680190:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320353.680599:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 555us (4838us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320353.680621:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1663 00000100:00100000:1.0:1767320353.680904:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320353.681325:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676416 00000100:00100000:3.0:1767320353.681333:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676416 00000100:00100000:3.0:1767320353.681372:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2199 00000100:00100000:3.0:1767320353.681380:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915676416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320353.681468:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320353.681532:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915676416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 150us (634us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320353.681539:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2199 00010000:00010000:2.0:1767320353.924706:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320354.953482:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320355.268662:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983048832:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320355.268769:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.268909:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048832 00000100:00100000:1.0:1767320355.268923:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048832 00000100:00100000:1.0:1767320355.269036:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116 00000100:00100000:1.0:1767320355.269048:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983048832:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320355.269058:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320355.269086:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320355.269173:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983048832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 129us (405us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320355.269181:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116 00000100:00100000:0.0:1767320355.269247:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983048832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320355.271923:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983048960:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320355.272036:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320355.272400:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983048960 00000100:00100000:2.0:1767320355.272414:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983048960 00000100:00100000:2.0:1767320355.272489:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1117 00000100:00100000:2.0:1767320355.272501:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983048960:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320355.272545:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320355.272614:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983048960:12345-0@lo:13:osp-pre-0-1.0 Request processed in 115us (578us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320355.272623:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1117 00000100:00100000:3.0:1767320355.272706:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983048960:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320355.272813:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320355.780588:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320355.780603:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320355.780771:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983049088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320355.780884:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320355.781080:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049088 00000100:00100000:2.0:1767320355.781091:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049088 00000100:00100000:2.0:1767320355.781171:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1664 00000100:00100000:2.0:1767320355.781182:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983049088:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320355.781298:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320355.781950:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320355.781957:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320355.781962:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320355.781969:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5599 00000100:00080000:3.0:1767320355.781974:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320355.781982:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320355.782018:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8a80 x1853163983049216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320355.782063:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320355.782067:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320355.782069:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320355.782074:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5599 00000100:00080000:3.0:1767320355.782076:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320355.782081:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320355.782094:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9180 x1853163983049472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320355.782104:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320355.782107:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320355.782109:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320355.782113:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5599 00000100:00080000:3.0:1767320355.782115:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320355.782119:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320355.782131:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ead80 x1853163983049600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320355.782311:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983049856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.782373:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.782456:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049856 00000100:00100000:1.0:1767320355.782464:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049856 00000100:00100000:1.0:1767320355.782508:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035 00000100:00100000:1.0:1767320355.782517:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983049856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.782567:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983049856:12345-0@lo:400:kworker.0 Request processed in 50us (193us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320355.782576:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035 00000100:00100000:1.0:1767320355.782608:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983049216:0@lo:38:kworker.0 00000100:00100000:1.0:1767320355.782642:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.782665:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983049472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320355.782688:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.782705:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983049600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320355.782771:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.782824:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049216 00000100:00100000:1.0:1767320355.782828:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049216 00000100:00100000:1.0:1767320355.782850:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049472 00000100:00100000:1.0:1767320355.782853:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049472 00000100:00100000:1.0:1767320355.782871:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2200 00000100:00100000:1.0:1767320355.782877:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983049216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320355.782903:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320355.782941:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983049216:12345-0@lo:38:kworker.0 Request processed in 62us (298us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320355.782949:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2200 00000100:00100000:1.0:1767320355.782961:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2201 00000100:00100000:1.0:1767320355.782965:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983049472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320355.782974:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320355.782998:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983049472:12345-0@lo:38:kworker.0 Request processed in 32us (310us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320355.783006:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2201 00000100:00100000:1.0:1767320355.783025:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049600 00000100:00100000:1.0:1767320355.783030:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049600 00000100:00100000:1.0:1767320355.783052:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 575 00000100:00100000:1.0:1767320355.783057:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983049600:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320355.783067:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320355.783095:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983049600:12345-0@lo:38:kworker.0 Request processed in 37us (324us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320355.783102:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 575 00000100:00100000:1.0:1767320355.783156:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983049856:0@lo:400:kworker.0 00000100:00080000:1.0:1767320355.783188:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320355.783194:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320355.783199:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983049216:0@lo:38:kworker.0 00000100:00080000:1.0:1767320355.783207:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320355.783210:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320355.783213:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983049472:0@lo:38:kworker.0 00000100:00080000:1.0:1767320355.783221:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320355.783224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320355.783228:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983049600:0@lo:38:kworker.0 00000100:00100000:1.0:1767320355.783304:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983049728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.783336:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.783357:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320355.783364:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983049344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.783389:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320355.783413:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049728 00000100:00100000:1.0:1767320355.783418:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049728 00000100:00100000:1.0:1767320355.783443:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036 00000100:00100000:1.0:1767320355.783449:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983049728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.783488:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983049728:12345-0@lo:400:kworker.0 Request processed in 39us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320355.783495:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036 00000100:00100000:1.0:1767320355.783510:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049344 00000100:00100000:1.0:1767320355.783514:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049344 00000100:00100000:1.0:1767320355.783549:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665 00000100:00100000:1.0:1767320355.783555:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983049344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.783587:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983049344:12345-0@lo:400:kworker.0 Request processed in 32us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320355.783595:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665 00000100:00080000:1.0:1767320355.783614:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320355.783638:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ebb80 x1853163983049344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320371 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320355.783651:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983049344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320355.783662:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983049728:0@lo:400:kworker.0 00000100:00100000:2.0:1767320355.784962:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983049088:12345-0@lo:400:kworker.0 Request processed in 3769us (4065us total) trans 0 rc 0/0 00010000:00080000:0.0:1767320355.784969:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8e00 x1853163983049088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320371 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320355.784982:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1664 00000100:00100000:0.0:1767320355.784993:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983049088:0@lo:400:kworker.0 00000100:00080000:3.0:1767320355.785167:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320355.785521:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320355.785526:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320355.785533:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5599 00000100:00080000:3.0:1767320355.785538:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320355.785545:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320355.785596:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb480 x1853163983049984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320355.785699:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983049984:0@lo:38:kworker.0 00000100:00100000:1.0:1767320355.785885:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320355.786012:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983049984 00000100:00100000:2.0:1767320355.786021:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983049984 00000100:00100000:2.0:1767320355.786108:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2202 00000100:00100000:2.0:1767320355.786116:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983049984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320355.786146:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320355.786232:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983049984:12345-0@lo:38:kworker.0 Request processed in 115us (348us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320355.786240:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2202 00000100:00080000:3.0:1767320355.786327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320355.786333:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320355.786337:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983049984:0@lo:38:kworker.0 00010000:00010000:2.0:1767320355.973025:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320356.996678:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320358.025393:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320358.795795:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320358.795904:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676672 00000100:00100000:2.0:1767320358.795919:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676672 00000100:00100000:2.0:1767320358.795996:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2203 00000100:00100000:2.0:1767320358.796007:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915676672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320358.796040:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320358.796130:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915676672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 123us (430us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320358.796136:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2203 00000100:00100000:0.0:1767320358.796487:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320358.796545:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915676800 00000100:00100000:1.0:1767320358.796557:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915676800 00000100:00100000:1.0:1767320358.796625:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1666 00000100:00100000:1.0:1767320358.796634:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767320358.796654:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320358.796699:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915676800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 67us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320358.796705:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1666 00010000:00010000:2.0:1767320359.046090:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320360.068560:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320360.389004:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983050112:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320360.389073:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320360.389404:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050112 00000100:00100000:3.0:1767320360.389413:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050112 00000100:00100000:3.0:1767320360.389473:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1767320360.389483:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983050112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320360.389514:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320360.389556:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983050112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (480us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320360.389563:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118 00000100:00080000:1.0:1767320360.389789:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320360.390316:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983050112:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320360.392442:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983050240:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320360.392516:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320360.392645:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050240 00000100:00100000:3.0:1767320360.392654:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050240 00000100:00100000:3.0:1767320360.392754:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767320360.392765:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983050240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320360.392799:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320360.392989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320360.394454:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983050240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1680us (1926us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320360.394469:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119 00000100:00100000:1.0:1767320360.394584:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983050240:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320360.903781:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320360.903794:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320360.903898:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983050368:0@lo:400:kworker.0 00000100:00080000:2.0:1767320360.903928:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320360.903934:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320360.903938:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320360.903943:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5604 00000100:00080000:2.0:1767320360.903947:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320360.903954:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320360.903969:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320360.903979:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ca80 x1853163983050496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320360.904017:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320360.904021:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320360.904023:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320360.904028:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5604 00000100:00080000:2.0:1767320360.904030:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320360.904034:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320360.904046:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398c000 x1853163983050752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320360.904057:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320360.904060:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320360.904062:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320360.904065:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5604 00000100:00080000:2.0:1767320360.904067:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320360.904071:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320360.904080:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320360.904083:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398dc00 x1853163983050880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320360.904092:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983050624:0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.904121:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320360.904139:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983051008:0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.904165:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320360.904167:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050368 00000100:00100000:1.0:1767320360.904175:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050368 00000100:00100000:0.0:1767320360.904216:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051008 00000100:00100000:0.0:1767320360.904221:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051008 00000100:00100000:1.0:1767320360.904232:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050624 00000100:00100000:1.0:1767320360.904237:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050624 00000100:00100000:0.0:1767320360.904259:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037 00000100:00100000:1.0:1767320360.904268:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1667 00000100:00100000:0.0:1767320360.904296:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983051008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320360.904314:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983050368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.904355:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983051008:12345-0@lo:400:kworker.0 Request processed in 86us (187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320360.904363:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037 00000100:00100000:1.0:1767320360.904372:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983050368:12345-0@lo:400:kworker.0 Request processed in 60us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320360.904380:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1667 00000100:00100000:0.0:1767320360.904389:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668 00000100:00100000:0.0:1767320360.904398:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983050624:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320360.904403:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320360.904430:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983050624:12345-0@lo:400:kworker.0 Request processed in 33us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320360.904436:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668 00010000:00080000:1.0:1767320360.904448:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398df80 x1853163983050368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320376 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320360.904462:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983050368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.904469:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983051008:0@lo:400:kworker.0 00010000:00080000:0.0:1767320360.904482:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398d500 x1853163983050624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320376 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320360.904491:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983050624:0@lo:400:kworker.0 00000100:00100000:1.0:1767320360.904502:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983050496:0@lo:38:kworker.0 00000100:00100000:1.0:1767320360.904533:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320360.904576:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983050752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320360.904577:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050496 00000100:00100000:0.0:1767320360.904583:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050496 00000100:00100000:1.0:1767320360.904604:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320360.904610:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2204 00000100:00100000:0.0:1767320360.904616:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983050496:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320360.904628:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983050880:0@lo:38:kworker.0 00010000:00020000:0.0:1767320360.904637:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320360.904653:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320360.904689:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983050496:12345-0@lo:38:kworker.0 Request processed in 72us (155us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320360.904690:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320360.904698:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320360.904715:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983050496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320360.904716:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2204 00000100:00100000:0.0:1767320360.904734:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050752 00000100:00100000:0.0:1767320360.904739:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050752 00000100:00100000:1.0:1767320360.904750:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983050880 00000100:00100000:1.0:1767320360.904755:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983050880 00000100:00100000:0.0:1767320360.904759:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2205 00000100:00100000:0.0:1767320360.904764:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983050752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320360.904775:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320360.904781:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 576 00000100:00100000:1.0:1767320360.904787:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983050880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320360.904798:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320360.904806:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983050752:12345-0@lo:38:kworker.0 Request processed in 40us (201us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320360.904813:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2205 00000100:00100000:1.0:1767320360.904830:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983050880:12345-0@lo:38:kworker.0 Request processed in 42us (177us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320360.904837:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 576 00000100:00080000:1.0:1767320360.904864:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320360.904868:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320360.904871:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983050752:0@lo:38:kworker.0 00000100:00080000:1.0:1767320360.904878:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320360.904881:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320360.904883:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983050880:0@lo:38:kworker.0 00000100:00080000:2.0:1767320360.906875:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320360.906883:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320360.906887:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320360.906894:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5604 00000100:00080000:2.0:1767320360.906899:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320360.906914:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320360.906929:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983051136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320360.906940:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f480 x1853163983051264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320360.906989:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320360.907051:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051136 00000100:00100000:1.0:1767320360.907059:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051136 00000100:00100000:0.0:1767320360.907092:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983051264:0@lo:38:kworker.0 00000100:00100000:1.0:1767320360.907103:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038 00000100:00100000:1.0:1767320360.907111:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983051136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.907133:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320360.907166:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983051136:12345-0@lo:400:kworker.0 Request processed in 55us (176us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320360.907175:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038 00000100:00100000:0.0:1767320360.907188:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051264 00000100:00100000:0.0:1767320360.907194:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051264 00000100:00100000:1.0:1767320360.907215:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983051136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320360.907223:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2206 00000100:00100000:0.0:1767320360.907230:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983051264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320360.907248:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320360.907302:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983051264:12345-0@lo:38:kworker.0 Request processed in 72us (169us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320360.907308:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2206 00000100:00080000:1.0:1767320360.907352:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320360.907357:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320360.907360:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983051264:0@lo:38:kworker.0 00010000:00010000:1.0:1767320361.092567:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320362.116451:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767320362.893561:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915676928 offset 0 length 520: 4 00000100:00100000:1.0:1767320362.893607:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320362.893649:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320362.894326:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677056 00000100:00100000:0.0:1767320362.894335:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677056 00000100:00100000:0.0:1767320362.894378:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677184 00000100:00100000:0.0:1767320362.894381:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677184 00000100:00100000:0.0:1767320362.894437:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1669 00000100:00100000:0.0:1767320362.894447:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915677184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767320362.894461:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320362.894498:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2207 00000100:00100000:3.0:1767320362.894506:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915677056:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:0.0:1767320362.894526:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915677184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (874us total) trans 0 rc 0/0 00010000:00020000:3.0:1767320362.894528:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320362.894538:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1669 00000100:00100000:3.0:1767320362.894564:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915677056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (955us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320362.894571:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2207 00010000:00010000:2.0:1767320363.143954:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320364.164735:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320365.188947:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320365.508602:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320365.508860:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320365.508930:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983051392:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320365.508989:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320365.509779:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983051520:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320365.509966:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320365.516071:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051392 00000100:00100000:2.0:1767320365.516089:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051392 00000100:00100000:2.0:1767320365.516183:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051520 00000100:00100000:2.0:1767320365.516187:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051520 00000100:00100000:2.0:1767320365.516211:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120 00000100:00100000:2.0:1767320365.516221:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983051392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320365.516262:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320365.516367:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983051392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 147us (7376us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320365.516374:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120 00000100:00100000:2.0:1767320365.516390:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121 00000100:00100000:2.0:1767320365.516397:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983051520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320365.516414:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320365.516445:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983051520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (6479us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320365.516451:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121 00000100:00100000:2.0:1767320365.516504:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983051392:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320365.516770:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320365.516775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320365.516801:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983051520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320366.020386:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320340/real 1767320340] req@ffff934609af1500 x1853163983045376/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320365 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320366.020416:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934609af1500 x1853163983045376/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320365 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320366.020434:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320366.020468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320366.020472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983045376:0@lo:250:kworker.0 00000100:00080000:3.0:1767320366.022003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320366.022014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320366.022018:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320366.022026:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5588 00000100:00080000:3.0:1767320366.022030:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320366.022037:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320366.022077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9500 x1853163983051648/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320366.022151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320366.022155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320366.022156:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320366.022160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5609 00000100:00080000:3.0:1767320366.022164:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320366.022167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320366.022182:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163983051904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320366.022204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320366.022207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320366.022209:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320366.022212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5609 00000100:00080000:3.0:1767320366.022215:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320366.022218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320366.022229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9f80 x1853163983052160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320366.022237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320366.022240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320366.022242:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320366.022245:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5609 00000100:00080000:3.0:1767320366.022248:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320366.022251:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320366.022266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163983052288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320366.022499:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320366.022520:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983052032:0@lo:400:kworker.0 00000100:00080000:3.0:1767320366.022524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320366.022529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320366.022531:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320366.022536:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5609 00000100:00080000:3.0:1767320366.022539:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320366.022544:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320366.022559:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163983052416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320366.022594:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320366.022628:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983051776:0@lo:400:kworker.0 00000100:00100000:3.0:1767320366.022663:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320366.022696:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052032 00000100:00100000:1.0:1767320366.022705:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052032 00000100:00080000:3.0:1767320366.022742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320366.022756:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1670 00000100:00100000:1.0:1767320366.022768:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983052032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320366.022771:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051776 00000100:00100000:3.0:1767320366.022776:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051776 00000100:00100000:3.0:1767320366.022815:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1671 00000100:00100000:1.0:1767320366.022823:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983052032:12345-0@lo:400:kworker.0 Request processed in 57us (229us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320366.022825:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983051776:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320366.022833:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1670 00010000:00080000:1.0:1767320366.022869:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb100 x1853163983052032/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320382 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320366.022874:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983051776:12345-0@lo:400:kworker.0 Request processed in 50us (209us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320366.022884:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983052032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320366.022886:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1671 00000100:00100000:0.0:1767320366.023002:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983051648:0@lo:250:kworker.0 00000400:00000100:0.0:1767320366.023024:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983051648 offset 0 length 520: 4 00000100:00100000:0.0:1767320366.023038:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983051904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320366.023052:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320366.023065:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983052160:0@lo:38:kworker.0 00000100:00100000:0.0:1767320366.023078:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320366.024002:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320366.024043:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8700 x1853163983051776/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320382 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320366.024056:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983051776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320366.027428:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983052288:0@lo:38:kworker.0 00000100:00100000:0.0:1767320366.027517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320366.027538:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983052416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320366.027565:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320366.027670:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052288 00000100:00100000:1.0:1767320366.027679:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052288 00000100:00100000:1.0:1767320366.027740:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 577 00000100:00100000:1.0:1767320366.027749:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983052288:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320366.027774:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320366.027821:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983052288:12345-0@lo:38:kworker.0 Request processed in 71us (303us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320366.027828:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 577 00000100:00080000:0.0:1767320366.028533:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320366.028540:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320366.028545:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983052288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320366.028866:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983051904 00000100:00100000:2.0:1767320366.028873:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983051904 00000100:00100000:2.0:1767320366.028912:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052160 00000100:00100000:2.0:1767320366.028915:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052160 00000100:00100000:2.0:1767320366.028933:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052416 00000100:00100000:2.0:1767320366.028936:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052416 00000100:00100000:2.0:1767320366.028948:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2208 00000100:00100000:2.0:1767320366.028954:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983051904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320366.028971:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320366.029019:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983051904:12345-0@lo:38:kworker.0 Request processed in 64us (5964us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320366.029026:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2208 00000100:00100000:2.0:1767320366.029043:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2209 00000100:00100000:2.0:1767320366.029047:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983052160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320366.029056:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320366.029081:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983052160:12345-0@lo:38:kworker.0 Request processed in 33us (6001us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320366.029086:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2209 00000100:00100000:2.0:1767320366.029098:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2210 00000100:00100000:2.0:1767320366.029102:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983052416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320366.029111:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320366.029133:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983052416:12345-0@lo:38:kworker.0 Request processed in 30us (1568us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320366.029138:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2210 00000100:00080000:0.0:1767320366.029221:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320366.029226:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320366.029230:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983051904:0@lo:38:kworker.0 00000100:00080000:0.0:1767320366.029240:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320366.029244:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320366.029247:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983052160:0@lo:38:kworker.0 00000100:00080000:0.0:1767320366.029257:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320366.029261:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320366.029263:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983052416:0@lo:38:kworker.0 00010000:00010000:2.0:1767320366.218299:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320367.245704:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320368.015898:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320368.015955:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320368.016024:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677312 00000100:00100000:1.0:1767320368.016034:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677312 00000100:00100000:1.0:1767320368.016076:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677440 00000100:00100000:1.0:1767320368.016079:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677440 00000100:00100000:1.0:1767320368.016132:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1672 00000100:00100000:1.0:1767320368.016141:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915677440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320368.016203:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915677440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 64us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320368.016210:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1672 00000100:00100000:1.0:1767320368.016226:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2211 00000100:00100000:1.0:1767320368.016230:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915677312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320368.016251:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320368.016303:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915677312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (385us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320368.016309:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2211 00000100:00080000:3.0:1767320368.016451:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320368.260621:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320369.284536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320370.308789:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320370.628577:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320370.628604:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983052544:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320370.628671:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320370.628772:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052544 00000100:00100000:0.0:1767320370.628781:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052544 00000100:00100000:0.0:1767320370.628852:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122 00000100:00100000:0.0:1767320370.628863:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983052544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320370.628902:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320370.628956:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983052544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320370.628964:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122 00000100:00100000:0.0:1767320370.629011:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983052544:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320370.629197:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320370.629443:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320370.629454:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983052672:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320370.629489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320370.629568:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052672 00000100:00100000:2.0:1767320370.629574:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052672 00000100:00100000:2.0:1767320370.629623:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1123 00000100:00100000:2.0:1767320370.629630:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983052672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320370.629647:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320370.629680:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983052672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320370.629686:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1123 00000100:00080000:3.0:1767320370.629737:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320370.629761:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983052672:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320371.140591:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.140601:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320371.140672:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.140678:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.140682:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.140689:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.140693:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320371.140700:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320371.140721:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396df80 x1853163983052928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320371.140762:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.140766:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.140768:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.140772:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.140774:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320371.140778:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320371.140789:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163983053184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320371.140798:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.140801:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.140802:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.140805:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.140808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320371.140811:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320371.140821:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f800 x1853163983053312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320371.140831:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320371.140834:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320371.140836:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320371.140839:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5614 00000100:00080000:2.0:1767320371.140841:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320371.140844:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320371.140853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163983053440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320371.140919:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320371.140934:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983052800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.141002:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320371.141058:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983053056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.141082:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320371.141117:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052800 00000100:00100000:0.0:1767320371.141124:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052800 00000100:00100000:0.0:1767320371.141187:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053056 00000100:00100000:0.0:1767320371.141191:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053056 00000100:00100000:0.0:1767320371.141214:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1673 00000100:00100000:0.0:1767320371.141223:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983052800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.141315:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983052800:12345-0@lo:400:kworker.0 Request processed in 92us (312us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320371.141323:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1673 00000100:00100000:0.0:1767320371.141340:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1674 00000100:00100000:0.0:1767320371.141346:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983053056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320371.141377:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983053056:12345-0@lo:400:kworker.0 Request processed in 31us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320371.141383:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1674 00000100:00080000:0.0:1767320371.141399:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320371.141424:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396f100 x1853163983052800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320387 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320371.141436:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983052800:0@lo:400:kworker.0 00010000:00080000:0.0:1767320371.141459:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163983053056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320387 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320371.141467:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983053056:0@lo:400:kworker.0 00000100:00100000:1.0:1767320371.141748:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983052928:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.141781:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.141811:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983053184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.141831:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.141853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983053312:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.141874:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.141891:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983053440:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.141911:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320371.141940:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053312 00000100:00100000:1.0:1767320371.141944:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053312 00000100:00100000:1.0:1767320371.141966:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 578 00000100:00100000:1.0:1767320371.141971:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983053312:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.141988:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.142022:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983053312:12345-0@lo:38:kworker.0 Request processed in 50us (146us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.142028:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 578 00000100:00080000:1.0:1767320371.142055:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320371.142059:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320371.142062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983053312:0@lo:38:kworker.0 00000100:00100000:1.0:1767320371.142087:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983052928 00000100:00100000:1.0:1767320371.142089:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983052928 00000100:00100000:1.0:1767320371.142101:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053184 00000100:00100000:1.0:1767320371.142104:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053184 00000100:00100000:1.0:1767320371.142110:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053440 00000100:00100000:1.0:1767320371.142112:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053440 00000100:00100000:1.0:1767320371.142121:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2212 00000100:00100000:1.0:1767320371.142126:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983052928:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320371.142223:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2213 00000100:00100000:0.0:1767320371.142228:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983053184:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320371.142291:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2214 00000100:00100000:2.0:1767320371.142297:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983053440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320371.142462:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320371.142507:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983052928:12345-0@lo:38:kworker.0 Request processed in 379us (725us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320371.142522:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2212 00000100:00080000:3.0:1767320371.143366:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320371.143374:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320371.143377:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983052928:0@lo:38:kworker.0 00010000:00020000:0.0:1767320371.148792:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320371.149005:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983053184:12345-0@lo:38:kworker.0 Request processed in 6770us (7167us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320371.149019:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2213 00010000:00020000:2.0:1767320371.151561:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320371.151658:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983053440:12345-0@lo:38:kworker.0 Request processed in 9356us (9742us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320371.151672:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2214 00000100:00080000:3.0:1767320371.153720:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320371.153731:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320371.153736:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983053184:0@lo:38:kworker.0 00000100:00080000:3.0:1767320371.153752:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320371.153756:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320371.153759:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983053440:0@lo:38:kworker.0 00010000:00010000:2.0:1767320371.336125:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320372.356598:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320373.140715:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320373.140767:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320373.140862:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677568 00000100:00100000:2.0:1767320373.140872:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677568 00000100:00100000:2.0:1767320373.140915:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677696 00000100:00100000:2.0:1767320373.140918:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677696 00000100:00100000:2.0:1767320373.140973:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1675 00000100:00100000:2.0:1767320373.140983:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915677696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320373.141049:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2215 00000100:00100000:0.0:1767320373.141055:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915677568:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767320373.141063:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915677696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320373.141069:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1675 00010000:00020000:0.0:1767320373.141075:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320373.141103:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915677568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (400us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320373.141109:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2215 00000100:00080000:3.0:1767320373.141175:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320373.383038:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320374.404758:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320375.428552:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320375.748704:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320375.748730:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983053568:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320375.748804:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320375.749297:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053568 00000100:00100000:0.0:1767320375.749308:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053568 00000100:00080000:2.0:1767320375.749436:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320375.749539:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124 00000100:00100000:0.0:1767320375.749548:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983053568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320375.749576:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320375.749626:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983053568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (822us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320375.749633:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124 00000100:00100000:2.0:1767320375.749706:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983053568:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320375.752170:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983053696:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320375.752239:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320375.756728:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053696 00000100:00100000:1.0:1767320375.756741:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053696 00000100:00100000:1.0:1767320375.756905:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125 00000100:00100000:1.0:1767320375.756916:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983053696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320375.756953:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320375.757027:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983053696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (4788us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320375.757033:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125 00000100:00100000:2.0:1767320375.757120:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983053696:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320375.757140:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320376.261299:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.261310:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320376.261370:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.261374:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.261378:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.261386:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.261391:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320376.261397:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320376.261429:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ca80 x1853163983053952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.261461:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.261464:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.261466:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.261469:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.261472:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320376.261475:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320376.261488:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d180 x1853163983054208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.261502:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.261505:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.261507:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.261510:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5619 00000100:00080000:2.0:1767320376.261513:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320376.261516:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320376.261527:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163983054336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320376.261536:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320376.261539:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320376.261541:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320376.261543:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5619 00000100:00100000:3.0:1767320376.261544:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983053824:0@lo:400:kworker.0 00000100:00080000:2.0:1767320376.261546:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320376.261549:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320376.261557:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d880 x1853163983054464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320376.261589:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983054080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320376.261609:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320376.261622:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983053952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320376.261642:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320376.261659:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983054208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320376.261672:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320376.261710:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053824 00000100:00100000:3.0:1767320376.261710:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320376.261716:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053824 00000100:00100000:3.0:1767320376.261724:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983054336:0@lo:38:kworker.0 00000100:00100000:3.0:1767320376.261748:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320376.261774:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983054464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320376.261779:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983053952 00000100:00100000:2.0:1767320376.261782:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983053952 00000100:00100000:2.0:1767320376.261792:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054080 00000100:00100000:2.0:1767320376.261794:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054080 00000100:00100000:3.0:1767320376.261794:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320376.261807:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054208 00000100:00100000:2.0:1767320376.261810:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054208 00000100:00100000:2.0:1767320376.261817:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054464 00000100:00100000:2.0:1767320376.261820:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054464 00000100:00080000:3.0:1767320376.261820:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320376.261830:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676 00000100:00100000:2.0:1767320376.261837:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983053824:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320376.262180:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983053824:12345-0@lo:400:kworker.0 Request processed in 342us (569us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320376.273726:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677 00000100:00100000:3.0:1767320376.273758:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983054080:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.273965:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2216 00000100:00100000:0.0:1767320376.273974:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983053952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320376.273994:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320376.274050:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983053952:12345-0@lo:38:kworker.0 Request processed in 76us (12406us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320376.274058:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2216 00000100:00100000:0.0:1767320376.274089:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2217 00000100:00100000:0.0:1767320376.274094:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983054208:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320376.274102:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983054080:12345-0@lo:400:kworker.0 Request processed in 362us (12429us total) trans 0 rc 0/0 00010000:00020000:0.0:1767320376.274103:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320376.274109:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677 00000100:00100000:3.0:1767320376.274126:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2218 00000100:00100000:0.0:1767320376.274128:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983054208:12345-0@lo:38:kworker.0 Request processed in 33us (12418us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320376.274130:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983054464:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320376.274133:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2217 00010000:00020000:3.0:1767320376.274140:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320376.274162:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983054464:12345-0@lo:38:kworker.0 Request processed in 31us (12368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320376.274168:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2218 00010000:00080000:3.0:1767320376.274194:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396fb80 x1853163983054080/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320392 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320376.274195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320376.274199:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320376.274203:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983053952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320376.274206:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983054080:0@lo:400:kworker.0 00000100:00080000:0.0:1767320376.274212:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320376.274215:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320376.274217:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983054208:0@lo:38:kworker.0 00000100:00080000:0.0:1767320376.274225:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320376.274228:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320376.274231:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983054464:0@lo:38:kworker.0 00000100:00100000:0.0:1767320376.274263:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054336 00010000:00080000:3.0:1767320376.274265:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ce00 x1853163983053824/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320392 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320376.274290:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983053824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320376.274292:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054336 00000100:00100000:0.0:1767320376.274321:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 579 00000100:00100000:0.0:1767320376.274326:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983054336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320376.274335:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320376.274376:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983054336:12345-0@lo:38:kworker.0 Request processed in 48us (12626us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320376.274382:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 579 00000100:00080000:3.0:1767320376.275630:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320376.275638:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320376.275642:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983054336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320376.283111:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676 00010000:00010000:1.0:1767320376.454172:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320377.480194:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320378.254159:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320378.254310:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320378.262223:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677952 00000100:00100000:2.0:1767320378.262240:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677952 00000100:00080000:0.0:1767320378.262432:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320378.263421:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915677824 00000100:00100000:2.0:1767320378.263431:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915677824 00000100:00100000:3.0:1767320378.269967:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1678 00000100:00100000:3.0:1767320378.269989:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915677952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320378.270096:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915677952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 118us (15948us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320378.270102:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1678 00000100:00100000:3.0:1767320378.270122:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2219 00000100:00100000:3.0:1767320378.270125:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915677824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320378.270150:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320378.270170:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915677824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (15862us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320378.270174:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2219 00010000:00010000:2.0:1767320378.501799:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320379.524530:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320380.548472:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320380.868513:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320380.868548:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983054592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320380.868621:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320380.869297:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054592 00000100:00100000:2.0:1767320380.869306:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054592 00000100:00080000:3.0:1767320380.869380:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320380.869392:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1126 00000100:00100000:2.0:1767320380.869402:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983054592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320380.869428:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320380.869476:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983054592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (855us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320380.869483:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1126 00000100:00100000:3.0:1767320380.869585:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983054592:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320380.869607:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320380.869620:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983054720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320380.869651:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320380.869900:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054720 00000100:00100000:3.0:1767320380.869904:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054720 00000100:00100000:3.0:1767320380.869948:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127 00000100:00080000:0.0:1767320380.869953:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320380.869958:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983054720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320380.869979:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320380.870014:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983054720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (362us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320380.870021:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127 00000100:00100000:0.0:1767320380.870046:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983054720:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320381.380854:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320381.380865:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320381.380931:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320381.380936:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320381.380940:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320381.380949:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5624 00000100:00080000:3.0:1767320381.380955:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320381.380964:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767320381.380971:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320381.380984:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983054848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320381.380995:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163983054976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320381.381045:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320381.381050:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1767320381.381051:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983054976:0@lo:38:kworker.0 00000100:00080000:3.0:1767320381.381053:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320381.381061:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320381.381062:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5624 00000100:00080000:3.0:1767320381.381068:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320381.381072:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320381.381105:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320381.381106:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1880 x1853163983055232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320381.381120:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320381.381124:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320381.381126:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320381.381130:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5624 00000100:00080000:3.0:1767320381.381132:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320381.381136:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320381.381152:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163983055360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320381.381160:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983055232:0@lo:38:kworker.0 00000100:00080000:3.0:1767320381.381164:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320381.381168:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320381.381170:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320381.381174:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5624 00000100:00080000:3.0:1767320381.381177:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320381.381181:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320381.381187:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320381.381195:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0700 x1853163983055488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320381.381207:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983055360:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.381222:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320381.381234:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320381.381235:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983055104:0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.381261:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983055488:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.381293:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320381.381462:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054848 00000100:00100000:1.0:1767320381.381471:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054848 00000100:00100000:1.0:1767320381.381527:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983054976 00000100:00100000:1.0:1767320381.381531:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983054976 00000100:00100000:1.0:1767320381.381541:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055232 00000100:00100000:1.0:1767320381.381544:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055232 00000100:00100000:1.0:1767320381.381551:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055104 00000100:00100000:1.0:1767320381.381553:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055104 00000100:00100000:1.0:1767320381.381571:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1679 00000100:00100000:1.0:1767320381.381578:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983054848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.382750:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320381.382758:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680 00000100:00100000:3.0:1767320381.382772:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983055104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320381.382789:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055360 00000100:00100000:0.0:1767320381.382797:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055360 00000100:00100000:3.0:1767320381.382829:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983055104:12345-0@lo:400:kworker.0 Request processed in 61us (1535us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320381.382832:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 580 00000100:00100000:3.0:1767320381.382841:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680 00000100:00100000:0.0:1767320381.382842:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983055360:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.382857:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055488 00000100:00100000:3.0:1767320381.382861:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055488 00010000:00020000:0.0:1767320381.382864:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320381.382875:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2220 00000100:00100000:3.0:1767320381.382879:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983054976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320381.382887:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320381.382905:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983055360:12345-0@lo:38:kworker.0 Request processed in 63us (1671us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320381.382914:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 580 00000100:00100000:3.0:1767320381.382918:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983054976:12345-0@lo:38:kworker.0 Request processed in 37us (1813us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320381.382923:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2220 00000100:00100000:3.0:1767320381.382935:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2221 00000100:00100000:3.0:1767320381.382939:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983055232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320381.382945:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320381.382951:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320381.382957:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320381.382961:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983055360:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.382977:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983055232:12345-0@lo:38:kworker.0 Request processed in 34us (1789us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320381.382978:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320381.382985:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320381.382991:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983055232:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.382997:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2221 00000100:00080000:0.0:1767320381.383003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320381.383009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320381.383014:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983054976:0@lo:38:kworker.0 00000100:00100000:3.0:1767320381.383015:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2222 00000100:00080000:0.0:1767320381.383052:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320381.383293:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983055488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320381.383303:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:2.0:1767320381.384130:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb100 x1853163983055104/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320397 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320381.384145:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983055104:0@lo:400:kworker.0 00000100:00100000:3.0:1767320381.385484:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983055488:12345-0@lo:38:kworker.0 Request processed in 2185us (2732us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320381.385505:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2222 00000100:00080000:0.0:1767320381.385670:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320381.385676:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320381.385680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983055488:0@lo:38:kworker.0 00000100:00100000:1.0:1767320381.386104:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983054848:12345-0@lo:400:kworker.0 Request processed in 4524us (5045us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320381.386114:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1679 00010000:00080000:2.0:1767320381.386151:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eaa00 x1853163983054848/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320397 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320381.386163:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983054848:0@lo:400:kworker.0 00010000:00010000:2.0:1767320381.572788:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320382.597046:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320383.393118:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320383.394210:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320383.396407:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678208 00000100:00100000:0.0:1767320383.396425:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678208 00000100:00100000:0.0:1767320383.396570:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678080 00000100:00100000:0.0:1767320383.396575:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678080 00000100:00100000:0.0:1767320383.396611:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1681 00000100:00100000:0.0:1767320383.396631:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915678208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320383.396733:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915678208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 105us (3626us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320383.396748:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1681 00000100:00100000:0.0:1767320383.396766:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2223 00000100:00100000:0.0:1767320383.396770:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915678080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320383.396790:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320383.396818:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915678080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (2615us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320383.396828:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2223 00000100:00080000:2.0:1767320383.398881:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320383.620642:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320384.645130:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320385.669114:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320385.988557:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320385.988578:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983055616:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320385.988651:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320385.989186:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320385.989199:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983055744:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320385.989249:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320386.007157:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055616 00000100:00100000:0.0:1767320386.007178:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055616 00000100:00100000:0.0:1767320386.007327:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055744 00000100:00100000:0.0:1767320386.007331:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055744 00000100:00100000:0.0:1767320386.007354:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128 00000100:00100000:0.0:1767320386.007364:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983055616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320386.007399:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320386.007491:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983055616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (18840us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320386.007513:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128 00000100:00100000:0.0:1767320386.007530:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1129 00000100:00100000:0.0:1767320386.007536:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983055744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320386.007552:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320386.007585:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983055744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (18342us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320386.007590:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1129 00000100:00100000:0.0:1767320386.007652:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983055616:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320386.010563:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320386.010575:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320386.010654:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983055744:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320386.500952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320386.500964:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320386.501034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320386.501039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320386.501043:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320386.501051:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5629 00000100:00080000:0.0:1767320386.501056:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320386.501064:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320386.501084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb100 x1853163983056000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320386.501129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320386.501132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320386.501134:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320386.501139:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5629 00000100:00080000:0.0:1767320386.501141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320386.501146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320386.501161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964700 x1853163983056256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320386.501170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320386.501174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320386.501176:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320386.501179:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5629 00000100:00080000:0.0:1767320386.501182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320386.501185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320386.501197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163983056384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320386.501219:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320386.501222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320386.501260:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320386.501265:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5629 00000100:00080000:0.0:1767320386.501267:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320386.501289:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320386.501303:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198aa00 x1853163983056768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320386.501329:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983056512:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.501393:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320386.501451:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320386.501458:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983056128:0@lo:400:kworker.0 00000100:00100000:0.0:1767320386.501482:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320386.501648:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983055872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.501678:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320386.501713:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320386.501719:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983056640:0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.501745:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320386.501775:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056512 00000100:00100000:1.0:1767320386.501781:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056512 00000100:00100000:1.0:1767320386.501820:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056640 00000100:00100000:1.0:1767320386.501823:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056640 00000100:00100000:1.0:1767320386.501844:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1039 00000100:00100000:1.0:1767320386.501853:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983056512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.501908:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983056512:12345-0@lo:400:kworker.0 Request processed in 56us (515us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320386.501916:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1039 00000100:00100000:1.0:1767320386.501944:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040 00000100:00100000:1.0:1767320386.501949:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604773050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983056640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.501980:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604773050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983056640:12345-0@lo:400:kworker.0 Request processed in 30us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320386.501985:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040 00000100:00100000:1.0:1767320386.502179:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056128 00000100:00100000:1.0:1767320386.502183:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056128 00000100:00100000:1.0:1767320386.502214:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983055872 00000100:00100000:1.0:1767320386.502216:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983055872 00000100:00100000:1.0:1767320386.502246:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1682 00000100:00100000:1.0:1767320386.502254:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983056128:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.502314:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983056128:12345-0@lo:400:kworker.0 Request processed in 61us (830us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320386.502321:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1682 00000100:00100000:1.0:1767320386.502334:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683 00000100:00100000:1.0:1767320386.502339:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983055872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.502364:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983055872:12345-0@lo:400:kworker.0 Request processed in 25us (685us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320386.502371:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683 00000100:00080000:1.0:1767320386.502396:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320386.502431:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983056640:0@lo:400:kworker.0 00010000:00080000:1.0:1767320386.502442:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eaa00 x1853163983055872/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320402 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320386.502457:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983055872:0@lo:400:kworker.0 00010000:00080000:1.0:1767320386.502489:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163983056128/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320402 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320386.502498:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983056128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320386.502508:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983056512:0@lo:400:kworker.0 00000100:00100000:2.0:1767320386.502714:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983056000:0@lo:38:kworker.0 00000100:00100000:2.0:1767320386.502749:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320386.502788:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983056256:0@lo:38:kworker.0 00000100:00100000:2.0:1767320386.502812:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320386.502831:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983056384:0@lo:38:kworker.0 00000100:00100000:2.0:1767320386.502855:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320386.502878:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983056768:0@lo:38:kworker.0 00000100:00100000:2.0:1767320386.502897:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320386.502936:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056384 00000100:00100000:2.0:1767320386.502940:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056384 00000100:00100000:2.0:1767320386.502960:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 581 00000100:00100000:2.0:1767320386.502964:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983056384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320386.502980:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320386.503017:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983056384:12345-0@lo:38:kworker.0 Request processed in 51us (161us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320386.503024:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 581 00000100:00100000:2.0:1767320386.503040:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056000 00000100:00100000:2.0:1767320386.503044:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056000 00000100:00100000:2.0:1767320386.503058:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056256 00000100:00100000:2.0:1767320386.503061:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056256 00000100:00100000:2.0:1767320386.503068:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056768 00000100:00100000:2.0:1767320386.503070:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056768 00000100:00100000:2.0:1767320386.503079:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2224 00000100:00100000:2.0:1767320386.503084:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056000:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320386.503093:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320386.503117:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056000:12345-0@lo:38:kworker.0 Request processed in 31us (367us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320386.503122:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2224 00000100:00100000:2.0:1767320386.503132:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2225 00000100:00100000:2.0:1767320386.503135:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320386.503144:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320386.503166:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056256:12345-0@lo:38:kworker.0 Request processed in 28us (353us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320386.503171:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2225 00000100:00100000:2.0:1767320386.503181:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2226 00000100:00100000:2.0:1767320386.503184:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056768:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320386.503192:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320386.503213:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983056768:12345-0@lo:38:kworker.0 Request processed in 27us (315us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320386.503218:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2226 00000100:00080000:2.0:1767320386.503259:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320386.503265:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320386.503268:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983056000:0@lo:38:kworker.0 00000100:00080000:2.0:1767320386.503307:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320386.503311:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320386.503314:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983056256:0@lo:38:kworker.0 00000100:00080000:2.0:1767320386.503322:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320386.503325:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320386.503328:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983056384:0@lo:38:kworker.0 00000100:00080000:2.0:1767320386.503334:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320386.503337:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320386.503340:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983056768:0@lo:38:kworker.0 00010000:00010000:2.0:1767320386.692461:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320387.716540:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320388.492117:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320388.492513:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320388.513313:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678336 00000100:00100000:1.0:1767320388.513332:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678336 00000100:00100000:1.0:1767320388.513391:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678464 00000100:00100000:1.0:1767320388.513395:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678464 00000100:00100000:1.0:1767320388.513465:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1684 00000100:00100000:1.0:1767320388.513478:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915678464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320388.513572:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915678464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 96us (21059us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320388.513581:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1684 00000100:00100000:1.0:1767320388.513600:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2227 00000100:00100000:1.0:1767320388.513605:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915678336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320388.513625:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320388.513653:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915678336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (21550us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320388.513660:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2227 00000100:00080000:3.0:1767320388.513788:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320388.747378:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320389.519710:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767320389.519752:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915678592 offset 0 length 520: 4 00000100:00100000:2.0:1767320389.519774:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320389.519859:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678848 00000100:00100000:1.0:1767320389.519868:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678848 00000100:00100000:1.0:1767320389.519920:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678720 00000100:00100000:1.0:1767320389.519923:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678720 00000100:00100000:1.0:1767320389.519940:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1685 00000100:00100000:1.0:1767320389.519965:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2228 00000100:00100000:1.0:1767320389.519970:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915678720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320389.519992:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320389.520047:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915678720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 76us (272us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320389.520054:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2228 00000100:00100000:1.0:1767320389.520143:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915678848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320389.520509:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915678848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 550us (807us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320389.520517:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1685 00010000:00010000:2.0:1767320389.764763:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320390.788995:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320391.108713:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983056896:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320391.108802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320391.108892:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983056896 00000100:00100000:1.0:1767320391.108901:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983056896 00000100:00100000:0.0:1767320391.109221:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983057024:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320391.109253:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320391.109426:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320391.113237:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057024 00000100:00100000:1.0:1767320391.113253:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057024 00000100:00100000:1.0:1767320391.113355:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130 00000100:00100000:1.0:1767320391.113366:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983056896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320391.113548:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320391.113647:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983056896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 283us (4844us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320391.113655:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130 00000100:00100000:1.0:1767320391.113675:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1131 00000100:00100000:1.0:1767320391.113682:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983057024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320391.113697:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320391.113731:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983057024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (4477us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320391.113738:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1131 00000100:00100000:1.0:1767320391.113803:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983057024:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320391.113847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983056896:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320391.113977:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320391.621424:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320366/real 1767320366] req@ffff9346395e9500 x1853163983051648/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320391 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320391.621466:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346395e9500 x1853163983051648/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320391 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320391.621500:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320391.621513:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320391.621524:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983051648:0@lo:250:kworker.0 00000100:00080000:1.0:1767320391.621914:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320391.621920:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320391.621924:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320391.621929:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5614 00000100:00080000:1.0:1767320391.621934:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320391.621940:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320391.621977:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ed80 x1853163983057152/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320391.622049:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320391.622052:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320391.622055:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320391.622058:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5635 00000100:00080000:1.0:1767320391.622061:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320391.622065:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320391.622083:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163983057408/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320391.622111:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320391.622114:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320391.622116:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320391.622119:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5635 00000100:00080000:1.0:1767320391.622122:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320391.622126:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320391.622148:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965c00 x1853163983057664/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320391.622157:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320391.622159:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320391.622161:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320391.622165:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5635 00000100:00080000:1.0:1767320391.622167:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320391.622171:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320391.622184:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163983057792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320391.622207:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320391.622210:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320391.622212:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320391.622215:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5635 00000100:00080000:1.0:1767320391.622217:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320391.622220:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320391.622231:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913b80 x1853163983058176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320391.622263:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983057920:0@lo:400:kworker.0 00000100:00100000:1.0:1767320391.622355:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.622501:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983057152:0@lo:250:kworker.0 00000100:00100000:3.0:1767320391.622579:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983058048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320391.622653:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983057280:0@lo:400:kworker.0 00000400:00000100:2.0:1767320391.627225:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983057152 offset 0 length 520: 4 00000100:00100000:2.0:1767320391.627301:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983057408:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.627351:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.627396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983057664:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.627419:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.627438:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983057792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.627469:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.627697:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983058176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.627723:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.627775:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057408 00000100:00100000:2.0:1767320391.627784:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057408 00000100:00100000:2.0:1767320391.627818:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057664 00000100:00100000:2.0:1767320391.627821:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057664 00000100:00100000:2.0:1767320391.627835:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058176 00000100:00100000:2.0:1767320391.627837:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058176 00000100:00100000:2.0:1767320391.627851:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2229 00000100:00100000:2.0:1767320391.627857:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983057408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320391.627879:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320391.627921:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983057408:12345-0@lo:38:kworker.0 Request processed in 63us (570us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320391.627929:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2229 00000100:00100000:2.0:1767320391.627945:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2230 00000100:00100000:2.0:1767320391.627949:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983057664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320391.627958:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320391.627982:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983057664:12345-0@lo:38:kworker.0 Request processed in 32us (562us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320391.627988:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2230 00000100:00100000:2.0:1767320391.627999:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2231 00000100:00100000:2.0:1767320391.628002:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983058176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320391.628011:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320391.628035:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983058176:12345-0@lo:38:kworker.0 Request processed in 31us (311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320391.628040:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2231 00000100:00080000:2.0:1767320391.628080:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320391.628085:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320391.628089:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983057408:0@lo:38:kworker.0 00000100:00080000:2.0:1767320391.628097:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320391.628099:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320391.628102:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983057664:0@lo:38:kworker.0 00000100:00080000:2.0:1767320391.628108:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320391.628110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320391.628112:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983058176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.628175:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983057536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.628200:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.628227:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057536 00000100:00100000:2.0:1767320391.628231:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057536 00000100:00100000:2.0:1767320391.628292:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686 00000100:00100000:2.0:1767320391.628299:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983057536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.628339:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983057536:12345-0@lo:400:kworker.0 Request processed in 41us (138us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320391.628345:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686 00010000:00080000:2.0:1767320391.628570:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964700 x1853163983057536/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320407 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320391.628582:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983057536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.628705:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057792 00000100:00100000:2.0:1767320391.628709:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057792 00000100:00100000:2.0:1767320391.628730:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 582 00000100:00100000:2.0:1767320391.628735:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983057792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320391.628746:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320391.628781:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983057792:12345-0@lo:38:kworker.0 Request processed in 45us (1312us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320391.628788:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 582 00000100:00080000:2.0:1767320391.628806:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320391.628825:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320391.628830:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320391.628833:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983057792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320391.628883:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057920 00000100:00100000:2.0:1767320391.628887:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057920 00000100:00100000:2.0:1767320391.628918:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041 00000100:00100000:2.0:1767320391.628925:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983057920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.628959:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983057920:12345-0@lo:400:kworker.0 Request processed in 35us (6605us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320391.628969:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041 00000100:00100000:3.0:1767320391.629175:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320391.630939:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058048 00000100:00100000:2.0:1767320391.630953:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058048 00000100:00100000:2.0:1767320391.631026:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1042 00000100:00100000:2.0:1767320391.631036:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983058048:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.631129:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983058048:12345-0@lo:400:kworker.0 Request processed in 94us (1953us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320391.631146:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1042 00000100:00100000:3.0:1767320391.631261:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983058048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320391.632756:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320391.633380:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983057920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.638962:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983057280 00000100:00100000:2.0:1767320391.638979:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983057280 00000100:00100000:2.0:1767320391.639126:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687 00000100:00100000:2.0:1767320391.639166:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983057280:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320391.639299:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983057280:12345-0@lo:400:kworker.0 Request processed in 165us (6547us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320391.639309:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687 00010000:00080000:1.0:1767320391.640331:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398df80 x1853163983057280/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320407 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320391.640347:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983057280:0@lo:400:kworker.0 00000100:00080000:0.0:1767320391.644955:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320391.814738:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320392.837642:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320393.860590:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320394.637743:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320394.638107:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320394.642194:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679104 00000100:00100000:2.0:1767320394.642212:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679104 00000100:00100000:2.0:1767320394.642328:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915678976 00000100:00100000:2.0:1767320394.642332:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915678976 00000100:00100000:2.0:1767320394.642361:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1688 00000100:00100000:2.0:1767320394.642373:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915679104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320394.642471:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915679104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (4737us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320394.642479:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1688 00000100:00100000:2.0:1767320394.642499:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2232 00000100:00100000:2.0:1767320394.642504:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915678976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320394.642525:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320394.642552:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915678976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (4446us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320394.642559:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2232 00000100:00080000:0.0:1767320394.642780:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320394.886327:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320395.908473:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320396.228523:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320396.228618:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320396.228640:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983058432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320396.228717:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320396.228885:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058432 00000100:00100000:2.0:1767320396.228894:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058432 00000100:00100000:2.0:1767320396.228966:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1132 00000100:00100000:2.0:1767320396.228975:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983058432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320396.229006:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320396.229063:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983058432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (337us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320396.229071:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1132 00000100:00080000:0.0:1767320396.229240:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320396.229315:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983058432:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320396.230500:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983058304:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320396.230640:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320396.231097:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058304 00000100:00100000:2.0:1767320396.231105:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058304 00000100:00100000:2.0:1767320396.231215:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1133 00000100:00100000:2.0:1767320396.231225:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983058304:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320396.231253:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320396.231329:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983058304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (688us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320396.231336:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1133 00000100:00100000:1.0:1767320396.231425:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983058304:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320396.231517:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320396.743518:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320396.743529:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320396.743601:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320396.743606:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320396.743610:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320396.743621:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320396.743627:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320396.743634:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320396.743669:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163983058688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320396.743717:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320396.743721:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320396.743722:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320396.743726:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320396.743732:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320396.743737:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320396.743754:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965c00 x1853163983058944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320396.743766:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320396.743772:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320396.743774:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320396.743778:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320396.743781:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320396.743788:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320396.743810:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965f80 x1853163983059072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320396.743862:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320396.743867:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320396.743869:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320396.743876:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5640 00000100:00080000:2.0:1767320396.743879:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320396.743891:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320396.743911:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934605ed7480 x1853163983059456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320396.744015:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983058560:0@lo:400:kworker.0 00000100:00100000:0.0:1767320396.744144:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744221:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320396.744230:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983059328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320396.744310:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744374:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983058688:0@lo:38:kworker.0 00000100:00100000:0.0:1767320396.744419:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744462:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983058944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320396.744502:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744534:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983059072:0@lo:38:kworker.0 00000100:00100000:0.0:1767320396.744590:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744635:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983059456:0@lo:38:kworker.0 00000100:00100000:0.0:1767320396.744679:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320396.744733:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058560 00000100:00100000:0.0:1767320396.744744:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058560 00000100:00100000:0.0:1767320396.744937:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058688 00000100:00100000:0.0:1767320396.744941:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058688 00000100:00100000:1.0:1767320396.747599:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058944 00000100:00100000:1.0:1767320396.747620:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058944 00000100:00100000:1.0:1767320396.747657:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059456 00000100:00100000:1.0:1767320396.747661:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059456 00000100:00100000:1.0:1767320396.747679:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689 00000100:00100000:1.0:1767320396.747713:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983058560:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.747830:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983058560:12345-0@lo:400:kworker.0 Request processed in 141us (3689us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320396.747838:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689 00000100:00100000:1.0:1767320396.747857:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2233 00000100:00100000:1.0:1767320396.747862:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983058688:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320396.747883:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320396.747923:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983058688:12345-0@lo:38:kworker.0 Request processed in 59us (3507us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320396.747929:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2233 00000100:00100000:1.0:1767320396.747940:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2234 00000100:00100000:1.0:1767320396.747944:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983058944:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320396.747953:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320396.747992:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983058944:12345-0@lo:38:kworker.0 Request processed in 46us (3492us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320396.748001:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2234 00000100:00100000:1.0:1767320396.748013:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2235 00000100:00100000:1.0:1767320396.748018:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983059456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320396.748026:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320396.748061:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983059456:12345-0@lo:38:kworker.0 Request processed in 43us (3385us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320396.748067:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2235 00010000:00080000:1.0:1767320396.748119:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396fb80 x1853163983058560/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320412 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320396.748134:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983058560:0@lo:400:kworker.0 00000100:00080000:1.0:1767320396.748912:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320396.748955:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320396.748960:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320396.748963:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983058688:0@lo:38:kworker.0 00000100:00080000:1.0:1767320396.748974:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320396.748978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320396.748980:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983058944:0@lo:38:kworker.0 00000100:00080000:1.0:1767320396.748989:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320396.748992:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320396.748995:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983059456:0@lo:38:kworker.0 00000100:00100000:1.0:1767320396.749689:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983059200:0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.749749:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320396.749791:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320396.749798:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983058816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.749819:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320396.749848:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983058816 00000100:00100000:1.0:1767320396.749855:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983058816 00000100:00100000:1.0:1767320396.749898:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1690 00000100:00100000:1.0:1767320396.749905:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983058816:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.749952:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983058816:12345-0@lo:400:kworker.0 Request processed in 46us (131us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320396.749959:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1690 00010000:00080000:1.0:1767320396.750009:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964700 x1853163983058816/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320412 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320396.750088:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983058816:0@lo:400:kworker.0 00000100:00080000:1.0:1767320396.750105:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320396.750126:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059328 00000100:00100000:1.0:1767320396.750130:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059328 00000100:00100000:0.0:1767320396.750202:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059072 00000100:00100000:0.0:1767320396.750208:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059072 00000100:00100000:0.0:1767320396.750230:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 583 00000100:00100000:0.0:1767320396.750235:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983059072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320396.750252:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320396.750307:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983059072:12345-0@lo:38:kworker.0 Request processed in 69us (5715us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320396.750313:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 583 00000100:00080000:0.0:1767320396.750340:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320396.750345:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320396.750348:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983059072:0@lo:38:kworker.0 00000100:00100000:1.0:1767320396.750566:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059200 00000100:00100000:1.0:1767320396.750570:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059200 00000100:00100000:1.0:1767320396.750599:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043 00000100:00100000:1.0:1767320396.750606:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983059200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.750645:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983059200:12345-0@lo:400:kworker.0 Request processed in 40us (895us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320396.750649:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044 00000100:00100000:1.0:1767320396.750653:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043 00000100:00100000:0.0:1767320396.750656:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983059328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320396.750682:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983059200:0@lo:400:kworker.0 00000100:00100000:0.0:1767320396.750692:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983059328:12345-0@lo:400:kworker.0 Request processed in 37us (6386us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320396.750697:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044 00000100:00100000:1.0:1767320396.750721:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983059328:0@lo:400:kworker.0 00010000:00010000:2.0:1767320396.932525:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320397.958017:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320398.980890:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320399.759867:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320399.759982:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679360 00000100:00100000:0.0:1767320399.760025:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679360 00000100:00100000:0.0:1767320399.760119:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1691 00000100:00100000:0.0:1767320399.760131:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915679360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320399.760217:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915679360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (362us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320399.760226:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1691 00000100:00080000:1.0:1767320399.760396:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320399.764136:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320399.764262:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679232 00000100:00100000:3.0:1767320399.764316:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679232 00000100:00100000:3.0:1767320399.764376:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2236 00000100:00100000:3.0:1767320399.764400:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915679232:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320399.764431:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320399.764514:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915679232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 114us (389us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320399.764521:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2236 00010000:00010000:2.0:1767320400.004670:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320401.033065:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320401.350052:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983059584:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320401.350151:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320401.350425:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983059712:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320401.350455:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320401.350542:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059584 00000100:00100000:0.0:1767320401.350551:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059584 00000100:00100000:0.0:1767320401.350610:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059712 00000100:00100000:0.0:1767320401.350614:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059712 00000100:00100000:0.0:1767320401.350634:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1134 00000100:00100000:0.0:1767320401.350642:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983059584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320401.350671:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320401.350717:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983059584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (566us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320401.350725:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1134 00000100:00100000:0.0:1767320401.350742:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1135 00000100:00100000:0.0:1767320401.350749:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983059712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320401.350768:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320401.350798:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983059712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320401.350804:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1135 00000100:00080000:1.0:1767320401.350862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320401.350866:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320401.350903:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983059712:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320401.350933:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983059584:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320401.863751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320401.863761:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320401.863834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320401.863839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320401.863843:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320401.863850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5645 00000100:00080000:1.0:1767320401.863855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320401.863872:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320401.863898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163983059968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320401.863932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320401.863935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320401.863937:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320401.863941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5645 00000100:00080000:1.0:1767320401.863944:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320401.863957:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320401.863971:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398df80 x1853163983060224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320401.863980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320401.863983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320401.863985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320401.863989:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5645 00000100:00080000:1.0:1767320401.863991:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320401.863994:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320401.864006:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51180 x1853163983060352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320401.864027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320401.864031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320401.864033:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320401.864036:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5645 00000100:00080000:1.0:1767320401.864039:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320401.864042:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320401.864052:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ed80 x1853163983060736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320401.864089:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983060480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320401.864156:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320401.864200:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320401.864206:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983060096:0@lo:400:kworker.0 00000100:00100000:1.0:1767320401.864237:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320401.864769:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983060608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320401.864813:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320401.864845:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320401.864852:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983059840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320401.864875:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320401.864915:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060096 00000100:00100000:3.0:1767320401.864925:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060096 00000100:00100000:3.0:1767320401.864995:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059840 00000100:00100000:3.0:1767320401.864998:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059840 00000100:00100000:3.0:1767320401.865023:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692 00000100:00100000:3.0:1767320401.865031:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983060096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320401.865085:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983060096:12345-0@lo:400:kworker.0 Request processed in 54us (845us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320401.865093:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692 00000100:00100000:3.0:1767320401.865111:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1693 00000100:00100000:3.0:1767320401.865117:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983059840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320401.865150:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983059840:12345-0@lo:400:kworker.0 Request processed in 33us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320401.865156:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1693 00000100:00080000:3.0:1767320401.865172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320401.865199:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965c00 x1853163983059840/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320417 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320401.865217:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983059840:0@lo:400:kworker.0 00010000:00080000:3.0:1767320401.865245:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396fb80 x1853163983060096/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320417 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320401.865254:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983060096:0@lo:400:kworker.0 00000100:00100000:1.0:1767320401.865575:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060480 00000100:00100000:1.0:1767320401.865581:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060480 00000100:00100000:1.0:1767320401.865622:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060608 00000100:00100000:1.0:1767320401.865626:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060608 00000100:00100000:1.0:1767320401.865647:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1045 00000100:00100000:1.0:1767320401.865654:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983060480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320401.865693:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983060480:12345-0@lo:400:kworker.0 Request processed in 40us (1537us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320401.865700:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1045 00000100:00100000:1.0:1767320401.865714:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046 00000100:00100000:1.0:1767320401.865719:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983060608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320401.865749:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983060608:12345-0@lo:400:kworker.0 Request processed in 30us (935us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320401.865755:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046 00000100:00100000:0.0:1767320401.866527:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983059968:0@lo:38:kworker.0 00000100:00100000:0.0:1767320401.866589:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320401.866627:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983060224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320401.866649:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320401.866678:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983060352:0@lo:38:kworker.0 00000100:00100000:0.0:1767320401.866704:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320401.866744:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983060736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320401.866769:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320401.866804:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060352 00000100:00100000:0.0:1767320401.866810:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060352 00000100:00100000:0.0:1767320401.866842:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 584 00000100:00100000:0.0:1767320401.866849:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983060352:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320401.866874:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320401.866915:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983060352:12345-0@lo:38:kworker.0 Request processed in 64us (209us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320401.866923:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 584 00000100:00080000:0.0:1767320401.866979:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320401.866985:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320401.866989:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983060352:0@lo:38:kworker.0 00000100:00100000:0.0:1767320401.867015:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983059968 00000100:00100000:0.0:1767320401.867018:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983059968 00000100:00100000:0.0:1767320401.867043:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060224 00000100:00100000:0.0:1767320401.867047:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060224 00000100:00100000:0.0:1767320401.867055:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060736 00000100:00100000:0.0:1767320401.867058:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060736 00000100:00100000:0.0:1767320401.867068:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2237 00000100:00100000:0.0:1767320401.867072:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983059968:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320401.867083:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320401.867113:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983059968:12345-0@lo:38:kworker.0 Request processed in 39us (525us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320401.867119:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2237 00000100:00100000:0.0:1767320401.867131:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2238 00000100:00100000:0.0:1767320401.867134:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983060224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320401.867144:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320401.867167:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983060224:12345-0@lo:38:kworker.0 Request processed in 31us (517us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320401.867172:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2238 00000100:00100000:0.0:1767320401.867182:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2239 00000100:00100000:0.0:1767320401.867186:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983060736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320401.867194:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320401.867217:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983060736:12345-0@lo:38:kworker.0 Request processed in 30us (448us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320401.867222:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2239 00000100:00080000:0.0:1767320401.867244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320401.867248:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320401.867251:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983059968:0@lo:38:kworker.0 00000100:00080000:0.0:1767320401.867258:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320401.867261:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320401.867264:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983060224:0@lo:38:kworker.0 00000100:00080000:0.0:1767320401.867316:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320401.867320:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320401.867323:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983060736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320401.867740:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983060608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320401.867768:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983060480:0@lo:400:kworker.0 00010000:00010000:2.0:1767320402.058592:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320403.076537:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320404.100499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320404.880930:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320404.880972:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320404.881045:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679616 00000100:00100000:2.0:1767320404.881054:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679616 00000100:00100000:2.0:1767320404.881119:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679488 00000100:00100000:2.0:1767320404.881122:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679488 00000100:00100000:2.0:1767320404.881134:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1694 00000100:00100000:2.0:1767320404.881141:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915679616:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320404.881221:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915679616:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320404.881227:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1694 00000100:00100000:2.0:1767320404.881242:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2240 00000100:00100000:2.0:1767320404.881245:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915679488:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320404.881263:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320404.881308:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915679488:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (335us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320404.881312:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2240 00000100:00080000:1.0:1767320404.881710:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320405.124524:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320406.150324:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320406.468487:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983060864:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320406.468574:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320406.468706:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060864 00000100:00100000:1.0:1767320406.468716:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060864 00000100:00100000:1.0:1767320406.468796:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136 00000100:00100000:1.0:1767320406.468806:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983060864:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320406.468843:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320406.468907:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983060864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (332us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320406.468916:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983060992:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320406.468917:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136 00000100:00100000:3.0:1767320406.468968:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320406.468973:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983060864:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320406.469020:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320406.469067:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983060992 00000100:00100000:1.0:1767320406.469071:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983060992 00000100:00100000:1.0:1767320406.469107:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137 00000100:00100000:1.0:1767320406.469113:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983060992:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320406.469128:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767320406.469183:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320406.474634:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983060992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 5228us (5373us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320406.474650:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137 00000100:00100000:3.0:1767320406.478103:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983060992:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320406.981674:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320406.981685:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320406.981748:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320406.981752:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320406.981756:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320406.981764:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320406.981769:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320406.981777:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320406.981799:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983061120:0@lo:400:kworker.0 00000100:00100000:2.0:1767320406.981803:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627100 x1853163983061248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320406.981838:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320406.981841:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320406.981843:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320406.981847:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320406.981849:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320406.981865:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320406.981879:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163983061504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320406.981889:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320406.981893:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320406.981894:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320406.981897:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5650 00000100:00100000:1.0:1767320406.981898:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767320406.981900:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320406.981906:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320406.981907:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320406.981915:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983061376:0@lo:400:kworker.0 00000100:00100000:2.0:1767320406.981925:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163983061632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320406.981936:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320406.981939:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320406.981941:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320406.981944:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5650 00000100:00080000:2.0:1767320406.981947:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1767320406.981951:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320406.981951:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320406.981963:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594700 x1853163983061760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320406.981999:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061120 00000100:00100000:3.0:1767320406.981999:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061376 00000100:00100000:1.0:1767320406.982007:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061120 00000100:00100000:3.0:1767320406.982008:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061376 00000100:00100000:1.0:1767320406.982064:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1695 00000100:00100000:2.0:1767320406.982094:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983061248:0@lo:38:kworker.0 00000100:00100000:2.0:1767320406.982135:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320406.982159:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983061504:0@lo:38:kworker.0 00000100:00100000:2.0:1767320406.982179:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320406.982187:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061248 00000100:00100000:2.0:1767320406.982191:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983061632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320406.982194:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061248 00000100:00100000:3.0:1767320406.982211:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061504 00000100:00100000:3.0:1767320406.982215:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061504 00000100:00100000:2.0:1767320406.982219:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320406.982230:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696 00000100:00100000:3.0:1767320406.982240:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983061376:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320406.982246:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983061760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320406.982293:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320406.982318:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061760 00000100:00100000:2.0:1767320406.982322:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061760 00000100:00100000:3.0:1767320406.982344:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983061376:12345-0@lo:400:kworker.0 Request processed in 105us (390us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320406.982345:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2241 00000100:00100000:2.0:1767320406.982354:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983061248:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320406.982359:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696 00010000:00020000:2.0:1767320406.982375:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320406.982381:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2242 00000100:00100000:3.0:1767320406.982388:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983061504:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320406.982401:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320406.982413:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983061248:12345-0@lo:38:kworker.0 Request processed in 58us (276us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320406.982421:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2241 00000100:00100000:3.0:1767320406.982436:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983061504:12345-0@lo:38:kworker.0 Request processed in 44us (253us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320406.982439:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2243 00000100:00100000:3.0:1767320406.982451:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2242 00000100:00100000:2.0:1767320406.982452:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983061760:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320406.982492:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061632 00000100:00100000:3.0:1767320406.982497:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061632 00000100:00100000:3.0:1767320406.982519:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 585 00000100:00100000:3.0:1767320406.982523:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983061632:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320406.982533:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320406.982557:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983061632:12345-0@lo:38:kworker.0 Request processed in 32us (338us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320406.982563:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 585 00000100:00080000:3.0:1767320406.982577:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320406.982614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320406.982619:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320406.982622:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983061248:0@lo:38:kworker.0 00000100:00080000:3.0:1767320406.982632:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320406.982636:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320406.982640:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983061504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320406.982650:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983061120:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320406.982651:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320406.982660:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320406.982665:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983061632:0@lo:38:kworker.0 00000100:00100000:1.0:1767320406.982698:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983061120:12345-0@lo:400:kworker.0 Request processed in 625us (796us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320406.982705:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1695 00010000:00080000:3.0:1767320406.982725:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934605ed7480 x1853163983061120/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320422 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:1.0:1767320406.982738:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625f80 x1853163983061376/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320422 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320406.982749:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934605ed7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983061120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320406.982754:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983061376:0@lo:400:kworker.0 00010000:00020000:2.0:1767320406.982792:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320406.982829:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983061760:12345-0@lo:38:kworker.0 Request processed in 380us (557us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320406.982835:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2243 00000100:00080000:3.0:1767320406.983097:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320406.983102:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320406.983105:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983061760:0@lo:38:kworker.0 00010000:00010000:1.0:1767320407.172563:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320408.196941:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320409.221318:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320410.003126:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320410.003176:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320410.003268:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679744 00000100:00100000:2.0:1767320410.003328:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679744 00000100:00100000:0.0:1767320410.003338:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915679872 00000100:00100000:0.0:1767320410.003345:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915679872 00000100:00100000:2.0:1767320410.003394:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1697 00000100:00100000:0.0:1767320410.003409:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2244 00000100:00100000:2.0:1767320410.003411:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915679872:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320410.003421:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915679744:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320410.003443:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320410.003616:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320410.003833:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934605ed7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915679744:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 413us (717us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320410.003841:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2244 00000100:00100000:2.0:1767320410.004903:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915679872:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1493us (1720us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320410.004915:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1697 00010000:00010000:1.0:1767320410.246776:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320411.268453:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320411.588526:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320411.588552:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983062016:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320411.588581:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983061888:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320411.588639:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320411.588745:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320411.588856:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062016 00000100:00100000:0.0:1767320411.588865:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062016 00000100:00100000:0.0:1767320411.588938:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983061888 00000100:00100000:0.0:1767320411.588942:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983061888 00000100:00100000:0.0:1767320411.588962:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1138 00000100:00100000:0.0:1767320411.588971:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983062016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320411.589005:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320411.589056:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983062016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (419us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320411.589063:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1138 00000100:00100000:0.0:1767320411.589078:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767320411.589084:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983061888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320411.589102:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320411.589150:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983061888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (398us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320411.589156:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767320411.589202:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983062016:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320411.589289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320411.589293:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320411.589457:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983061888:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320412.100428:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320412.100439:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320412.100512:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320412.100517:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320412.100521:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320412.100530:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5655 00000100:00080000:2.0:1767320412.100534:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320412.100541:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320412.100575:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb100 x1853163983062272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320412.100629:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320412.100633:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320412.100635:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320412.100639:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5655 00000100:00080000:2.0:1767320412.100641:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320412.100645:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320412.100657:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163983062528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320412.100666:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320412.100669:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320412.100670:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320412.100673:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5655 00000100:00080000:2.0:1767320412.100676:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320412.100679:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320412.100689:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163983062656/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320412.100698:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320412.100702:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320412.100703:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320412.100706:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5655 00000100:00080000:2.0:1767320412.100708:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320412.100711:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320412.100722:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1f80 x1853163983062784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320412.100752:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983062400:0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.100844:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320412.100925:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062400 00000100:00100000:2.0:1767320412.100932:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062400 00000100:00100000:2.0:1767320412.100982:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698 00000100:00100000:2.0:1767320412.100990:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983062400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320412.101039:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983062400:12345-0@lo:400:kworker.0 Request processed in 50us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320412.101045:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698 00010000:00080000:2.0:1767320412.101081:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163983062400/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320428 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320412.101094:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983062400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320412.101204:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983062272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320412.101234:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320412.101266:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983062528:0@lo:38:kworker.0 00000100:00100000:1.0:1767320412.101312:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320412.101330:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983062656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320412.101352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320412.101371:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983062784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320412.101391:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320412.101420:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062272 00000100:00100000:1.0:1767320412.101424:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062272 00000100:00100000:1.0:1767320412.101440:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062528 00000100:00100000:1.0:1767320412.101442:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062528 00000100:00100000:1.0:1767320412.101448:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062784 00000100:00100000:1.0:1767320412.101451:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062784 00000100:00100000:1.0:1767320412.101459:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2245 00000100:00100000:1.0:1767320412.101463:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062272:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320412.101480:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320412.101513:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062272:12345-0@lo:38:kworker.0 Request processed in 49us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320412.101519:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2245 00000100:00100000:1.0:1767320412.101531:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2246 00000100:00100000:1.0:1767320412.101535:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062528:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320412.101542:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320412.101561:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062528:12345-0@lo:38:kworker.0 Request processed in 26us (249us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320412.101566:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2246 00000100:00100000:1.0:1767320412.101574:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2247 00000100:00100000:1.0:1767320412.101578:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320412.101586:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320412.101602:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983062784:12345-0@lo:38:kworker.0 Request processed in 23us (211us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320412.101607:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2247 00000100:00080000:1.0:1767320412.101629:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320412.101633:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320412.101636:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983062272:0@lo:38:kworker.0 00000100:00080000:1.0:1767320412.101644:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320412.101647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320412.101650:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983062528:0@lo:38:kworker.0 00000100:00080000:1.0:1767320412.101658:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320412.101661:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320412.101663:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983062784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320412.101751:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983062144:0@lo:400:kworker.0 00000100:00100000:0.0:1767320412.101790:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320412.101829:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062144 00000100:00100000:0.0:1767320412.101832:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062144 00000100:00100000:0.0:1767320412.101857:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699 00000100:00100000:0.0:1767320412.101863:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983062144:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320412.101898:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983062144:12345-0@lo:400:kworker.0 Request processed in 35us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320412.101904:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699 00010000:00080000:0.0:1767320412.101926:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9500 x1853163983062144/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320428 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320412.101936:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983062144:0@lo:400:kworker.0 00000100:00080000:0.0:1767320412.103150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320412.103200:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983062656 00000100:00100000:0.0:1767320412.103209:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983062656 00000100:00100000:0.0:1767320412.103249:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 586 00000100:00100000:0.0:1767320412.103255:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983062656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320412.103288:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320412.103350:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983062656:12345-0@lo:38:kworker.0 Request processed in 93us (1995us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320412.103357:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 586 00000100:00080000:1.0:1767320412.103614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320412.103620:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320412.103624:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983062656:0@lo:38:kworker.0 00010000:00010000:2.0:1767320412.293381:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320413.317486:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320414.102335:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320414.102379:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915680000 offset 0 length 520: 4 00000100:00100000:3.0:1767320414.102410:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320414.102642:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680256 00000100:00100000:1.0:1767320414.102653:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680256 00000100:00100000:1.0:1767320414.102752:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680128 00000100:00100000:1.0:1767320414.102756:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680128 00000100:00100000:2.0:1767320414.102763:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1700 00000100:00100000:2.0:1767320414.102780:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915680256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320414.102893:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915680256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (640us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320414.104178:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320414.105504:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2248 00000100:00100000:1.0:1767320414.105517:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915680128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320414.105559:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320414.105640:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915680128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 123us (3228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320414.105648:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2248 00000100:00100000:2.0:1767320414.115035:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1700 00010000:00010000:2.0:1767320414.343175:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320415.364600:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320416.196482:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320391/real 1767320391] req@ffff93463398ed80 x1853163983057152/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320416 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320416.196528:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463398ed80 x1853163983057152/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320416 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320416.196543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320416.196571:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320416.196575:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983057152:0@lo:250:kworker.0 00000100:00080000:3.0:1767320416.196609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320416.196614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320416.196618:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320416.196623:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5640 00000100:00080000:3.0:1767320416.196627:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320416.196634:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320416.196673:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163983062912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320416.196743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320416.196747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320416.196749:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320416.196753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5660 00000100:00080000:3.0:1767320416.196756:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320416.196759:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320416.196776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163983063168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320416.196799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320416.196802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320416.196804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320416.196807:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5660 00000100:00080000:3.0:1767320416.196810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320416.196814:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320416.196824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91880 x1853163983063424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320416.196832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320416.196835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320416.196836:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320416.196839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5660 00000100:00080000:3.0:1767320416.196841:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320416.196844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320416.196855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93b80 x1853163983063552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320416.196876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320416.196880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320416.196881:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320416.196885:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5660 00000100:00080000:3.0:1767320416.196889:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320416.196892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320416.196904:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163983063936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320416.196977:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983063040:0@lo:400:kworker.0 00000100:00100000:2.0:1767320416.197051:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.197226:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320416.197232:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983063808:0@lo:400:kworker.0 00000100:00100000:2.0:1767320416.197263:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320416.197417:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063808 00000100:00100000:2.0:1767320416.197425:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063808 00000100:00100000:2.0:1767320416.197483:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047 00000100:00100000:2.0:1767320416.197491:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983063808:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320416.197540:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983063808:12345-0@lo:400:kworker.0 Request processed in 50us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320416.197548:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047 00000100:00080000:2.0:1767320416.197566:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320416.197594:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983063808:0@lo:400:kworker.0 00000100:00100000:1.0:1767320416.197726:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983062912:0@lo:250:kworker.0 00000400:00000100:1.0:1767320416.197751:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983062912 offset 0 length 520: 4 00000100:00100000:1.0:1767320416.197774:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983063168:0@lo:38:kworker.0 00000100:00100000:1.0:1767320416.197800:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320416.197825:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983063424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320416.197844:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320416.197859:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983063552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320416.197880:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320416.197898:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983063936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320416.197917:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320416.197945:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063552 00000100:00100000:1.0:1767320416.197950:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063552 00000100:00100000:1.0:1767320416.197972:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 587 00000100:00100000:1.0:1767320416.197977:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983063552:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320416.197994:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320416.198028:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983063552:12345-0@lo:38:kworker.0 Request processed in 49us (147us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320416.198034:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 587 00000100:00100000:1.0:1767320416.198052:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063040 00000100:00100000:1.0:1767320416.198055:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063040 00000100:00100000:1.0:1767320416.198082:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063168 00000100:00100000:1.0:1767320416.198100:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063168 00000100:00100000:3.0:1767320416.198239:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063424 00000100:00100000:3.0:1767320416.198245:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063424 00000100:00100000:3.0:1767320416.198259:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063936 00000100:00100000:3.0:1767320416.198262:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063936 00000100:00100000:3.0:1767320416.198303:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701 00000100:00100000:3.0:1767320416.198310:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983063040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320416.198349:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983063040:12345-0@lo:400:kworker.0 Request processed in 40us (1297us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320416.198354:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701 00000100:00100000:3.0:1767320416.198365:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2249 00000100:00100000:3.0:1767320416.198368:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983063424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320416.198377:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320416.198394:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983063424:12345-0@lo:38:kworker.0 Request processed in 24us (549us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320416.198398:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2249 00000100:00100000:3.0:1767320416.198408:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2250 00000100:00100000:3.0:1767320416.198410:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983063936:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320416.198417:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320416.198435:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983063936:12345-0@lo:38:kworker.0 Request processed in 22us (517us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320416.198439:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2250 00000100:00080000:3.0:1767320416.198475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320416.198482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320416.198485:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983063424:0@lo:38:kworker.0 00000100:00080000:3.0:1767320416.198493:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320416.198495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320416.198498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983063552:0@lo:38:kworker.0 00000100:00080000:3.0:1767320416.198503:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320416.198507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320416.198509:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983063936:0@lo:38:kworker.0 00000100:00080000:3.0:1767320416.198541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320416.198706:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983063296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320416.198737:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320416.198768:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320416.198774:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983063680:0@lo:400:kworker.0 00000100:00100000:0.0:1767320416.198797:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320416.198820:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063680 00000100:00100000:0.0:1767320416.198824:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063680 00000100:00100000:0.0:1767320416.198858:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048 00000100:00100000:0.0:1767320416.198865:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983063680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320416.198902:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983063680:12345-0@lo:400:kworker.0 Request processed in 38us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320416.198908:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048 00000100:00080000:0.0:1767320416.198923:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320416.198941:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983063680:0@lo:400:kworker.0 00000100:00100000:0.0:1767320416.198958:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983063296 00000100:00100000:0.0:1767320416.198961:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983063296 00000100:00100000:0.0:1767320416.198987:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1702 00000100:00100000:0.0:1767320416.198992:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983063296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320416.199028:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983063296:12345-0@lo:400:kworker.0 Request processed in 35us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320416.199033:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1702 00000100:00080000:0.0:1767320416.199045:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320416.199060:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965c00 x1853163983063296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320432 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320416.199071:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983063296:0@lo:400:kworker.0 00010000:00080000:2.0:1767320416.199384:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c627100 x1853163983063040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320432 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320416.199396:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983063040:0@lo:400:kworker.0 00000100:00100000:1.0:1767320416.199491:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2251 00000100:00100000:1.0:1767320416.199496:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983063168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320416.199509:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320416.199550:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983063168:12345-0@lo:38:kworker.0 Request processed in 52us (1749us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320416.199557:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2251 00000100:00080000:1.0:1767320416.199584:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320416.199588:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320416.199591:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983063168:0@lo:38:kworker.0 00010000:00010000:1.0:1767320416.388693:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320416.709799:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983064064:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320416.709872:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320416.710051:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983064192:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320416.710083:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320416.710158:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064064 00000100:00100000:1.0:1767320416.710166:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064064 00000100:00100000:1.0:1767320416.710213:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064192 00000100:00100000:1.0:1767320416.710217:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064192 00000100:00100000:1.0:1767320416.710235:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140 00000100:00100000:1.0:1767320416.710244:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983064064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320416.710302:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320416.710352:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983064064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (479us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320416.710360:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140 00000100:00100000:1.0:1767320416.710376:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1141 00000100:00100000:1.0:1767320416.710382:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983064192:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320416.710399:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320416.710442:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983064192:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320416.710449:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1141 00000100:00100000:0.0:1767320416.710615:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983064192:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320416.710858:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983064064:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320417.412656:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320418.436577:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320419.214304:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320419.214441:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320419.234546:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680512 00000100:00100000:2.0:1767320419.234565:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680512 00000100:00100000:2.0:1767320419.234720:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680384 00000100:00100000:2.0:1767320419.234724:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680384 00000100:00100000:2.0:1767320419.234747:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1703 00000100:00100000:2.0:1767320419.234757:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915680512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320419.234854:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915680512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (20592us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320419.234862:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1703 00000100:00100000:2.0:1767320419.234882:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2252 00000100:00100000:2.0:1767320419.234886:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915680384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320419.234907:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320419.234935:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915680384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (20494us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320419.234941:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2252 00000100:00080000:3.0:1767320419.235355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320419.462737:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320420.485438:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320421.319997:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.320009:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320421.320086:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.320090:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.320094:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.320103:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.320108:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.320117:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320421.320142:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435aa00 x1853163983064448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320421.320183:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.320186:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.320188:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.320192:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.320195:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.320199:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320421.320211:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359500 x1853163983064704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320421.320221:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.320224:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.320226:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.320229:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.320231:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.320235:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320421.320248:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435a300 x1853163983064832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320421.320299:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320421.320303:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320421.320305:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320421.320309:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5664 00000100:00080000:1.0:1767320421.320311:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320421.320315:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320421.320326:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163983065216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320421.320408:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983065088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320477:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320421.320523:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320421.320530:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983064320:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320556:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320421.320583:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983064576:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320619:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320421.320636:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320421.320642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983064960:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320661:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320421.320701:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064320 00000100:00100000:0.0:1767320421.320708:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064320 00000100:00100000:0.0:1767320421.320755:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064576 00000100:00100000:0.0:1767320421.320758:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064576 00000100:00100000:0.0:1767320421.320782:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704 00000100:00100000:0.0:1767320421.320790:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983064320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320840:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983064320:12345-0@lo:400:kworker.0 Request processed in 50us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320421.320847:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704 00000100:00100000:0.0:1767320421.320862:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1705 00000100:00100000:0.0:1767320421.320868:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983064576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320894:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983064576:12345-0@lo:400:kworker.0 Request processed in 27us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320421.320901:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1705 00000100:00100000:0.0:1767320421.320918:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065088 00000100:00100000:0.0:1767320421.320921:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065088 00000100:00100000:0.0:1767320421.320944:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064960 00000100:00100000:0.0:1767320421.320947:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064960 00000100:00100000:0.0:1767320421.320963:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049 00000100:00100000:0.0:1767320421.320968:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983065088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.320994:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983065088:12345-0@lo:400:kworker.0 Request processed in 25us (517us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320421.320999:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049 00000100:00100000:0.0:1767320421.321008:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050 00000100:00100000:0.0:1767320421.321013:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983064960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.321033:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983064960:12345-0@lo:400:kworker.0 Request processed in 20us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320421.321038:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050 00000100:00100000:0.0:1767320421.321079:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983064960:0@lo:400:kworker.0 00010000:00080000:0.0:1767320421.321090:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b800 x1853163983064576/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320437 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320421.321099:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983064576:0@lo:400:kworker.0 00010000:00080000:0.0:1767320421.321125:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396fb80 x1853163983064320/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320437 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320421.321132:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983064320:0@lo:400:kworker.0 00000100:00100000:0.0:1767320421.321139:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983065088:0@lo:400:kworker.0 00000100:00080000:3.0:1767320421.321313:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320421.321318:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320421.321320:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320421.321332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983064448:0@lo:38:kworker.0 00000100:00100000:3.0:1767320421.321360:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320421.321379:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983064704:0@lo:38:kworker.0 00000100:00100000:3.0:1767320421.321399:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320421.321412:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983064832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320421.321432:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320421.321446:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983065216:0@lo:38:kworker.0 00000100:00100000:3.0:1767320421.321465:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320421.321485:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064448 00000100:00100000:3.0:1767320421.321488:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064448 00000100:00100000:3.0:1767320421.321503:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064704 00000100:00100000:3.0:1767320421.321505:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064704 00000100:00100000:3.0:1767320421.321512:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065216 00000100:00100000:3.0:1767320421.321514:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065216 00000100:00100000:3.0:1767320421.321524:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2253 00000100:00100000:3.0:1767320421.321530:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983064448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320421.321547:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320421.321579:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983064448:12345-0@lo:38:kworker.0 Request processed in 48us (218us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320421.321585:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2253 00000100:00100000:3.0:1767320421.321606:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2254 00000100:00100000:3.0:1767320421.321610:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983064704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320421.321618:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320421.321641:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983064704:12345-0@lo:38:kworker.0 Request processed in 30us (242us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320421.321647:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2254 00000100:00100000:3.0:1767320421.321657:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2255 00000100:00100000:3.0:1767320421.321660:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983065216:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320421.321667:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320421.321688:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983065216:12345-0@lo:38:kworker.0 Request processed in 25us (221us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320421.321702:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2255 00000100:00080000:3.0:1767320421.321726:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320421.321733:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320421.321737:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983064448:0@lo:38:kworker.0 00000100:00080000:3.0:1767320421.321745:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320421.321749:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320421.321752:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983064704:0@lo:38:kworker.0 00000100:00080000:3.0:1767320421.321760:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320421.321763:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320421.321766:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983065216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320421.323354:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983064832 00000100:00100000:0.0:1767320421.323369:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983064832 00000100:00100000:0.0:1767320421.323425:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 588 00000100:00100000:0.0:1767320421.323433:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983064832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320421.323461:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320421.323544:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983064832:12345-0@lo:38:kworker.0 Request processed in 110us (2110us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320421.323551:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 588 00000100:00080000:3.0:1767320421.324228:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320421.324235:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320421.324239:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983064832:0@lo:38:kworker.0 00010000:00010000:2.0:1767320421.510473:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320421.828937:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320421.828961:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983065344:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320421.829035:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320421.829152:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065344 00000100:00100000:3.0:1767320421.829160:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065344 00000100:00100000:3.0:1767320421.829215:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1142 00000100:00100000:3.0:1767320421.829225:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983065344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320421.829256:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320421.829341:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983065344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 117us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320421.829348:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1142 00000100:00100000:1.0:1767320421.829435:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983065344:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320421.829964:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320421.829976:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983065472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320421.830010:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320421.830080:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065472 00000100:00100000:1.0:1767320421.830086:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065472 00000100:00100000:1.0:1767320421.830121:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143 00000100:00100000:1.0:1767320421.830129:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983065472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320421.830147:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320421.830188:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983065472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (177us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320421.830195:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143 00000100:00100000:2.0:1767320421.830306:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983065472:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320422.532511:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320423.558370:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320424.335921:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320424.336102:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680768 00000100:00100000:0.0:1767320424.336114:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680768 00000100:00100000:0.0:1767320424.336214:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1706 00000100:00100000:0.0:1767320424.336225:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915680768:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320424.336363:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915680768:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 140us (452us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320424.336371:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1706 00000100:00080000:3.0:1767320424.336827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320424.337210:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320424.337594:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680640 00000100:00100000:3.0:1767320424.337602:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680640 00000100:00100000:3.0:1767320424.337642:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2256 00000100:00100000:3.0:1767320424.337649:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915680640:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320424.337672:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320424.337843:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915680640:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 192us (633us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320424.337856:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2256 00010000:00010000:2.0:1767320424.585000:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320425.605648:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320426.436441:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.436453:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320426.436556:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.436560:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.436564:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.436591:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.436596:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.436603:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320426.436633:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320426.436642:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359500 x1853163983065728/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320426.436652:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983065600:0@lo:400:kworker.0 00000100:00080000:1.0:1767320426.436760:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.436778:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.436780:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.436806:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.436808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320426.436808:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1767320426.436812:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320426.436817:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983065856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.436824:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b480 x1853163983065984/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320426.436846:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.436849:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.436852:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.436886:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.436888:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.436894:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320426.436925:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b800 x1853163983066112/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320426.436965:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320426.436968:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320426.436970:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320426.436974:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5669 00000100:00080000:1.0:1767320426.436976:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320426.436980:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320426.437020:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246d80 x1853163983066496/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320426.437067:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320426.437076:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983066240:0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.437587:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.437625:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320426.437634:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983066368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.437661:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.437694:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066368 00000100:00100000:2.0:1767320426.437702:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066368 00000100:00100000:2.0:1767320426.437766:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051 00000100:00100000:2.0:1767320426.437775:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983066368:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.437823:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983066368:12345-0@lo:400:kworker.0 Request processed in 49us (160us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320426.437830:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051 00000100:00100000:2.0:1767320426.437872:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983066368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.437908:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983065728:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.437934:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320426.438070:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065856 00000100:00100000:0.0:1767320426.438075:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065856 00000100:00100000:0.0:1767320426.438104:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065728 00000100:00100000:0.0:1767320426.438107:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065728 00000100:00100000:0.0:1767320426.438123:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1707 00000100:00100000:0.0:1767320426.438130:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983065856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320426.438166:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983065856:12345-0@lo:400:kworker.0 Request processed in 38us (582us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320426.438173:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1707 00000100:00100000:0.0:1767320426.438184:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2257 00000100:00100000:0.0:1767320426.438188:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983065728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320426.438204:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320426.438226:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983065728:12345-0@lo:38:kworker.0 Request processed in 36us (292us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320426.438232:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2257 00010000:00080000:0.0:1767320426.438260:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435aa00 x1853163983065856/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320442 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320426.438296:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983065856:0@lo:400:kworker.0 00000100:00080000:0.0:1767320426.438313:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320426.438317:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320426.438795:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.438800:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.438803:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983065728:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.438812:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983065984:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.438844:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.438870:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983066112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320426.438893:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.438929:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983066496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320426.439074:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065984 00000100:00100000:0.0:1767320426.439080:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065984 00000100:00100000:0.0:1767320426.439105:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2258 00000100:00100000:0.0:1767320426.439110:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983065984:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320426.439123:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320426.439153:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983065984:12345-0@lo:38:kworker.0 Request processed in 42us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320426.439159:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2258 00000100:00100000:0.0:1767320426.439188:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066112 00000100:00100000:0.0:1767320426.439192:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066112 00000100:00100000:0.0:1767320426.439209:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 589 00000100:00100000:0.0:1767320426.439214:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983066112:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320426.439223:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320426.439244:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983066112:12345-0@lo:38:kworker.0 Request processed in 29us (351us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320426.439250:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 589 00000100:00100000:2.0:1767320426.439327:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320426.439616:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.439621:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.439625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983065984:0@lo:38:kworker.0 00000100:00080000:2.0:1767320426.439637:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.439640:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320426.439643:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983066112:0@lo:38:kworker.0 00000100:00100000:0.0:1767320426.439706:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066496 00000100:00100000:0.0:1767320426.439710:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066496 00000100:00100000:0.0:1767320426.439740:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2259 00000100:00100000:0.0:1767320426.439745:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983066496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320426.439755:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320426.439788:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983066496:12345-0@lo:38:kworker.0 Request processed in 42us (461us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320426.439794:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2259 00000100:00080000:2.0:1767320426.439861:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320426.439867:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320426.439872:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.439874:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983066496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320426.442981:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983065600 00000100:00100000:0.0:1767320426.442996:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983065600 00000100:00100000:0.0:1767320426.443164:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708 00000100:00100000:0.0:1767320426.443175:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983065600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320426.443291:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983065600:12345-0@lo:400:kworker.0 Request processed in 118us (3424us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320426.443298:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708 00010000:00080000:2.0:1767320426.443727:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435bb80 x1853163983065600/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320442 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320426.443744:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983065600:0@lo:400:kworker.0 00000100:00100000:1.0:1767320426.445025:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320426.445168:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066240 00000100:00100000:2.0:1767320426.445177:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066240 00000100:00080000:3.0:1767320426.445246:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320426.445247:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052 00000100:00080000:3.0:1767320426.445254:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320426.445260:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983066240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320426.445363:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983066240:12345-0@lo:400:kworker.0 Request processed in 106us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320426.445371:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052 00000100:00100000:3.0:1767320426.445565:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983066240:0@lo:400:kworker.0 00010000:00010000:2.0:1767320426.628616:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320426.949177:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320426.949202:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983066624:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320426.949298:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320426.949516:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066624 00000100:00100000:3.0:1767320426.949525:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066624 00000100:00100000:1.0:1767320426.950229:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983066752:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320426.950291:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320426.950796:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066752 00000100:00100000:3.0:1767320426.950803:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066752 00000100:00100000:3.0:1767320426.950835:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1144 00000100:00100000:3.0:1767320426.950844:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983066624:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320426.950889:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320426.950942:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983066624:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (1643us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320426.950950:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1144 00000100:00100000:3.0:1767320426.950967:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145 00000100:00100000:3.0:1767320426.950973:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983066752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320426.950989:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320426.951020:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983066752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (749us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320426.951025:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145 00000100:00100000:3.0:1767320426.951157:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983066624:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320426.951975:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983066752:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320427.653756:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320428.676824:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320429.451766:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320429.451930:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681024 00000100:00100000:2.0:1767320429.451941:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681024 00000100:00100000:2.0:1767320429.452041:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1709 00000100:00100000:2.0:1767320429.452052:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915681024:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320429.452152:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915681024:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (395us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320429.452160:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1709 00000100:00080000:3.0:1767320429.452242:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320429.457876:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320429.458185:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915680896 00000100:00100000:0.0:1767320429.458195:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915680896 00000100:00100000:0.0:1767320429.458255:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2260 00000100:00100000:0.0:1767320429.458263:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915680896:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320429.458313:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320429.458380:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915680896:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 116us (514us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320429.458387:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2260 00010000:00010000:2.0:1767320429.707177:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320430.727065:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320431.558320:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320431.558332:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320431.558398:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320431.558411:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320431.558415:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320431.558424:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5674 00000100:00080000:3.0:1767320431.558429:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320431.558435:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320431.558458:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b800 x1853163983067008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320431.558475:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320431.558496:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983066880:0@lo:400:kworker.0 00000100:00080000:3.0:1767320431.558520:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320431.558523:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320431.558524:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320431.558528:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5674 00000100:00080000:3.0:1767320431.558530:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320431.558535:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320431.558557:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543d180 x1853163983067264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320431.558562:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320431.558567:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320431.558570:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320431.558571:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320431.558574:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5674 00000100:00080000:3.0:1767320431.558576:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320431.558579:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320431.558591:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ed80 x1853163983067392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320431.558609:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320431.558612:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320431.558614:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320431.558617:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5674 00000100:00080000:3.0:1767320431.558619:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320431.558622:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983067648:0@lo:400:kworker.0 00000100:00080000:3.0:1767320431.558622:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320431.558630:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f480 x1853163983067776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320431.558646:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320431.558656:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983067136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.558680:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067648 00000100:00100000:3.0:1767320431.558682:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320431.558687:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067648 00000100:00100000:3.0:1767320431.558701:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320431.558706:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983067520:0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.558741:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053 00000100:00100000:0.0:1767320431.558751:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983067648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320431.559004:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983066880 00000100:00100000:1.0:1767320431.559010:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983066880 00000100:00100000:1.0:1767320431.559043:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067136 00000100:00100000:1.0:1767320431.559047:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067136 00000100:00100000:1.0:1767320431.559070:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1710 00000100:00100000:1.0:1767320431.559077:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983066880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320431.559134:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983066880:12345-0@lo:400:kworker.0 Request processed in 58us (572us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320431.559141:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1710 00000100:00100000:1.0:1767320431.559157:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1711 00000100:00100000:1.0:1767320431.559162:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983067136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320431.559187:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983067136:12345-0@lo:400:kworker.0 Request processed in 25us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320431.559192:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1711 00000100:00080000:1.0:1767320431.559219:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320431.559223:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:1.0:1767320431.559260:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988000 x1853163983066880/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320447 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320431.559299:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983066880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320431.559694:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767320431.559830:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543fb80 x1853163983067136/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320447 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320431.559842:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983067136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.559961:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983067648:12345-0@lo:400:kworker.0 Request processed in 1211us (1312us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320431.559968:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053 00000100:00100000:0.0:1767320431.559986:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067520 00000100:00100000:0.0:1767320431.559990:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067520 00000100:00100000:0.0:1767320431.560026:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1054 00000100:00100000:0.0:1767320431.560031:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983067520:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320431.560064:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983067520:12345-0@lo:400:kworker.0 Request processed in 32us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320431.560069:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1054 00000100:00080000:0.0:1767320431.560087:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320431.560106:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983067520:0@lo:400:kworker.0 00000100:00100000:3.0:1767320431.560145:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983067648:0@lo:400:kworker.0 00000100:00100000:2.0:1767320431.560261:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983067008:0@lo:38:kworker.0 00000100:00100000:2.0:1767320431.560313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320431.560337:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983067264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320431.560354:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320431.560376:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983067392:0@lo:38:kworker.0 00000100:00100000:2.0:1767320431.560397:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320431.560431:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983067776:0@lo:38:kworker.0 00000100:00100000:2.0:1767320431.560454:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320431.560480:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067008 00000100:00100000:2.0:1767320431.560484:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067008 00000100:00100000:2.0:1767320431.560503:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067264 00000100:00100000:2.0:1767320431.560505:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067264 00000100:00100000:2.0:1767320431.560512:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067776 00000100:00100000:2.0:1767320431.560514:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067776 00000100:00100000:2.0:1767320431.560523:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2261 00000100:00100000:2.0:1767320431.560528:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320431.560559:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320431.560590:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067008:12345-0@lo:38:kworker.0 Request processed in 61us (276us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320431.560600:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2261 00000100:00100000:2.0:1767320431.560612:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2262 00000100:00100000:2.0:1767320431.560616:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067264:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320431.560625:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320431.560644:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067264:12345-0@lo:38:kworker.0 Request processed in 27us (289us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320431.560649:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2262 00000100:00100000:2.0:1767320431.560658:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2263 00000100:00100000:2.0:1767320431.560662:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d7050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067776:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320431.560671:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320431.560688:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d7050 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983067776:12345-0@lo:38:kworker.0 Request processed in 24us (233us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320431.560693:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2263 00000100:00100000:0.0:1767320431.561201:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067392 00000100:00100000:0.0:1767320431.561209:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067392 00000100:00100000:0.0:1767320431.561237:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 590 00000100:00100000:0.0:1767320431.561242:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983067392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320431.561253:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320431.561310:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983067392:12345-0@lo:38:kworker.0 Request processed in 67us (911us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320431.561316:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 590 00000100:00080000:0.0:1767320431.561386:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.561392:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.561395:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983067008:0@lo:38:kworker.0 00000100:00080000:0.0:1767320431.561413:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.561417:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.561420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983067264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320431.561428:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.561431:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.561434:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983067392:0@lo:38:kworker.0 00000100:00080000:0.0:1767320431.561441:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320431.561445:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320431.561447:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983067776:0@lo:38:kworker.0 00010000:00010000:1.0:1767320431.748735:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320432.068542:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320432.068571:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983067904:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320432.068650:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320432.068668:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320432.068679:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983068032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320432.068706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320432.068731:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983067904 00000100:00100000:1.0:1767320432.068739:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983067904 00000100:00100000:1.0:1767320432.068795:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068032 00000100:00100000:1.0:1767320432.068799:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068032 00000100:00100000:1.0:1767320432.068817:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1146 00000100:00100000:1.0:1767320432.068826:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983067904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320432.068859:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320432.068904:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983067904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (254us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320432.068912:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1146 00000100:00100000:1.0:1767320432.068943:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1147 00000100:00100000:1.0:1767320432.068951:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983068032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320432.068966:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320432.068996:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983068032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (290us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320432.069002:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1147 00000100:00100000:1.0:1767320432.069042:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983067904:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320432.069221:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983068032:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320432.772799:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320433.796693:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320434.575076:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320434.575182:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681280 00000100:00100000:2.0:1767320434.575194:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681280 00000100:00100000:2.0:1767320434.575315:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1712 00000100:00100000:2.0:1767320434.575326:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915681280:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320434.575410:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915681280:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (343us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320434.575421:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1712 00000100:00080000:0.0:1767320434.575459:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320434.580996:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320434.582579:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681152 00000100:00100000:1.0:1767320434.582595:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681152 00000100:00100000:1.0:1767320434.582658:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2264 00000100:00100000:1.0:1767320434.582667:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915681152:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320434.582689:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320434.582763:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915681152:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (1778us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320434.582771:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2264 00010000:00010000:1.0:1767320434.820879:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320435.844564:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320436.676848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320436.676860:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320436.676931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320436.676936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320436.676942:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320436.676950:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5680 00000100:00080000:3.0:1767320436.676956:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320436.676962:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983068160:0@lo:400:kworker.0 00000100:00080000:3.0:1767320436.676964:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320436.676992:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f100 x1853163983068288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320436.677041:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320436.677043:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320436.677048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320436.677050:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320436.677055:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5680 00000100:00080000:3.0:1767320436.677057:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320436.677062:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320436.677074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c380 x1853163983068544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320436.677082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320436.677085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320436.677087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320436.677091:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5680 00000100:00080000:3.0:1767320436.677093:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320436.677097:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320436.677109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c000 x1853163983068672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320436.677130:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068160 00000100:00080000:3.0:1767320436.677137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320436.677138:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068160 00000100:00080000:3.0:1767320436.677144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320436.677147:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320436.677150:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5680 00000100:00080000:3.0:1767320436.677152:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320436.677168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320436.677189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e680 x1853163983069056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320436.677212:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1713 00000100:00100000:3.0:1767320436.677215:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983068416:0@lo:400:kworker.0 00000100:00100000:2.0:1767320436.677225:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983068160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320436.677250:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320436.677296:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320436.677304:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983068160:12345-0@lo:400:kworker.0 Request processed in 81us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320436.677305:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983068800:0@lo:400:kworker.0 00000100:00100000:2.0:1767320436.677327:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1713 00000100:00100000:3.0:1767320436.677350:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320436.677377:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068800 00000100:00100000:3.0:1767320436.677381:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068800 00000100:00100000:3.0:1767320436.677408:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055 00000100:00100000:3.0:1767320436.677414:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983068800:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320436.677433:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068416 00000100:00100000:2.0:1767320436.677438:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068416 00000100:00100000:2.0:1767320436.677466:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1714 00000100:00100000:2.0:1767320436.677472:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983068416:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.677548:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983068928:0@lo:400:kworker.0 00000100:00100000:1.0:1767320436.677628:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983068288:0@lo:38:kworker.0 00000100:00100000:2.0:1767320436.679874:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983068416:12345-0@lo:400:kworker.0 Request processed in 2388us (2610us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320436.679890:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1714 00000100:00080000:2.0:1767320436.679934:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320436.679939:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320436.679988:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543ea00 x1853163983068160/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320452 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320436.680003:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983068160:0@lo:400:kworker.0 00010000:00080000:2.0:1767320436.680056:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543df80 x1853163983068416/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320452 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320436.680072:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983068416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.682499:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320436.682591:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068928 00000100:00100000:0.0:1767320436.682600:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068928 00000100:00080000:2.0:1767320436.682684:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320436.682691:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056 00000100:00100000:0.0:1767320436.682702:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983068928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.682775:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983068928:12345-0@lo:400:kworker.0 Request processed in 75us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320436.682787:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056 00000100:00100000:2.0:1767320436.682820:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983068928:0@lo:400:kworker.0 00000100:00100000:1.0:1767320436.683923:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320436.683980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983068544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320436.683991:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068288 00000100:00100000:2.0:1767320436.683999:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068288 00000100:00100000:1.0:1767320436.684024:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320436.684046:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068544 00000100:00100000:2.0:1767320436.684051:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068544 00000100:00100000:1.0:1767320436.684059:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983068672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320436.684072:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2265 00000100:00100000:2.0:1767320436.684079:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983068288:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.684088:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767320436.684101:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320436.684117:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983069056:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.684151:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320436.684154:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983068288:12345-0@lo:38:kworker.0 Request processed in 73us (234us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320436.684163:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2265 00000100:00080000:1.0:1767320436.684298:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320436.684304:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320436.684318:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983068288:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.684343:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983068672 00000100:00100000:1.0:1767320436.684347:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983068672 00000100:00100000:1.0:1767320436.684368:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 591 00000100:00100000:1.0:1767320436.684372:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983068672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320436.684382:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320436.684410:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983068672:12345-0@lo:38:kworker.0 Request processed in 36us (322us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320436.684416:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 591 00000100:00100000:3.0:1767320436.687018:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983068800:12345-0@lo:400:kworker.0 Request processed in 9595us (9659us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320436.687032:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055 00000100:00100000:3.0:1767320436.687110:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983068800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320436.690518:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069056 00000100:00100000:0.0:1767320436.690535:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069056 00000100:00100000:1.0:1767320436.690570:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2266 00000100:00100000:1.0:1767320436.690581:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983068544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320436.690605:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320436.690703:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983068544:12345-0@lo:38:kworker.0 Request processed in 122us (6678us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320436.691205:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2267 00000100:00100000:0.0:1767320436.691212:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983069056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320436.691224:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320436.691262:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983069056:12345-0@lo:38:kworker.0 Request processed in 49us (7111us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320436.691283:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2267 00000100:00080000:2.0:1767320436.693288:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320436.693298:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320436.693303:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983068544:0@lo:38:kworker.0 00000100:00080000:2.0:1767320436.693328:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320436.693331:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320436.693334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983068672:0@lo:38:kworker.0 00000100:00080000:2.0:1767320436.693351:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320436.693354:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320436.693357:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983069056:0@lo:38:kworker.0 00000100:00100000:1.0:1767320436.694493:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2266 00010000:00010000:2.0:1767320436.868594:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320437.188832:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983069184:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320437.188876:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320437.188894:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983069312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320437.188920:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320437.188976:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320437.189023:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069184 00000100:00100000:2.0:1767320437.189031:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069184 00000100:00100000:2.0:1767320437.189086:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069312 00000100:00100000:2.0:1767320437.189089:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069312 00000100:00100000:2.0:1767320437.189109:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148 00000100:00100000:2.0:1767320437.189118:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983069184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320437.189159:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320437.189222:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983069184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320437.189229:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148 00000100:00100000:2.0:1767320437.189246:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1149 00000100:00100000:2.0:1767320437.189253:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983069312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320437.189288:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320437.189323:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983069184:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320437.189335:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983069312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (358us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320437.189342:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1149 00000100:00100000:3.0:1767320437.189587:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983069312:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320437.892756:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320438.916841:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320439.692072:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320439.692622:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681536 00000100:00100000:0.0:1767320439.692633:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681536 00000100:00100000:1.0:1767320439.692817:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320439.692853:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681408 00000100:00100000:1.0:1767320439.692857:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681408 00000100:00100000:1.0:1767320439.692892:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1715 00000100:00100000:1.0:1767320439.692905:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915681536:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:2.0:1767320439.692905:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320439.692929:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2268 00000100:00100000:2.0:1767320439.692937:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915681408:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320439.692959:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320439.692986:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915681536:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (922us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320439.692995:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1715 00000100:00100000:2.0:1767320439.692999:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915681408:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (181us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320439.693009:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2268 00000400:00000100:3.0:1767320439.693603:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915681664 offset 0 length 520: 4 00010000:00010000:2.0:1767320439.941563:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320440.964592:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320441.796372:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320416/real 1767320416] req@ffff93463c626680 x1853163983062912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320441 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320441.796401:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c626680 x1853163983062912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320441 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320441.796417:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320441.796438:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320441.796441:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983062912:0@lo:250:kworker.0 00000100:00080000:0.0:1767320441.796499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320441.796503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320441.796506:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320441.796509:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5664 00000100:00080000:0.0:1767320441.796512:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320441.796517:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320441.796544:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53480 x1853163983069440/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320441.796621:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983069440:0@lo:250:kworker.0 00000400:00000100:1.0:1767320441.796658:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983069440 offset 0 length 520: 4 00000100:00100000:2.0:1767320441.796853:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767320441.796889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320441.796892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320441.796894:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320441.796898:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5685 00000100:00080000:0.0:1767320441.796900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320441.796904:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320441.796918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52300 x1853163983069696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320441.796942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320441.796944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320441.796946:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320441.796949:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5685 00000100:00080000:0.0:1767320441.796950:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320441.796953:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320441.796961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50e00 x1853163983069952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320441.796967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320441.796970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320441.796971:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320441.796974:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5685 00000100:00080000:0.0:1767320441.796975:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320441.796978:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320441.796984:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51f80 x1853163983070080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320441.797032:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983070208:0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.797059:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.797082:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320441.797086:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983069824:0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.797101:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.797118:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983069696:0@lo:38:kworker.0 00000100:00100000:1.0:1767320441.797137:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.797155:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983069952:0@lo:38:kworker.0 00000100:00100000:1.0:1767320441.797173:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.797187:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983070080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320441.797201:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.797251:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069824 00000100:00100000:1.0:1767320441.797258:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069824 00000100:00100000:1.0:1767320441.797310:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069696 00000100:00100000:1.0:1767320441.797313:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069696 00000100:00100000:1.0:1767320441.797321:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069952 00000100:00100000:1.0:1767320441.797323:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069952 00000100:00100000:1.0:1767320441.797330:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716 00000100:00100000:1.0:1767320441.797336:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983069824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.797370:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983069824:12345-0@lo:400:kworker.0 Request processed in 34us (267us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320441.797375:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716 00000100:00100000:1.0:1767320441.797386:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2269 00000100:00100000:1.0:1767320441.797389:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983069696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320441.797402:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320441.797421:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983069696:12345-0@lo:38:kworker.0 Request processed in 31us (284us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320441.797425:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2269 00000100:00100000:1.0:1767320441.797431:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2270 00000100:00100000:1.0:1767320441.797434:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983069952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320441.797439:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320441.797450:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983069952:12345-0@lo:38:kworker.0 Request processed in 16us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320441.797454:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2270 00000100:00080000:1.0:1767320441.797465:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320441.797481:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f50a80 x1853163983069824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320457 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320441.797490:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983069824:0@lo:400:kworker.0 00000100:00080000:1.0:1767320441.797502:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320441.797505:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320441.797506:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983069696:0@lo:38:kworker.0 00000100:00080000:1.0:1767320441.797510:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320441.797512:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320441.797514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983069952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320441.797649:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070208 00000100:00100000:3.0:1767320441.797652:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070208 00000100:00100000:3.0:1767320441.797693:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057 00000100:00100000:3.0:1767320441.797700:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983070208:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.797746:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983070208:12345-0@lo:400:kworker.0 Request processed in 48us (685us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320441.797750:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057 00000100:00100000:3.0:1767320441.797768:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070080 00000100:00100000:3.0:1767320441.797771:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070080 00000100:00100000:3.0:1767320441.797783:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 592 00000100:00100000:3.0:1767320441.797786:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983070080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320441.797793:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320441.797810:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983070080:12345-0@lo:38:kworker.0 Request processed in 23us (608us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320441.797814:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 592 00000100:00100000:3.0:1767320441.797840:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983070208:0@lo:400:kworker.0 00000100:00080000:3.0:1767320441.797851:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320441.797853:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320441.797855:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983070080:0@lo:38:kworker.0 00000100:00080000:1.0:1767320441.797935:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320441.797970:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983070336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320441.797993:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320441.798011:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983069568:0@lo:400:kworker.0 00000100:00100000:2.0:1767320441.798023:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320441.798071:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983069568 00000100:00100000:1.0:1767320441.798074:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983069568 00000100:00100000:1.0:1767320441.798102:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1717 00000100:00100000:1.0:1767320441.798105:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983069568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320441.798146:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983069568:12345-0@lo:400:kworker.0 Request processed in 39us (121us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320441.798152:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1717 00000100:00080000:2.0:1767320441.798207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320441.798236:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f51500 x1853163983069568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320457 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320441.798246:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983069568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.798293:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070336 00000100:00100000:3.0:1767320441.798296:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070336 00000100:00100000:3.0:1767320441.798324:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058 00000100:00100000:3.0:1767320441.798328:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983070336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320441.798351:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983070336:12345-0@lo:400:kworker.0 Request processed in 23us (357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320441.798355:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058 00000100:00080000:2.0:1767320441.798413:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320441.798426:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983070336:0@lo:400:kworker.0 00000100:00080000:0.0:1767320441.798632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320441.798635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320441.798637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320441.798640:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5685 00000100:00080000:0.0:1767320441.798642:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320441.798645:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320441.798662:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f52a00 x1853163983070464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320441.798783:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983070464:0@lo:38:kworker.0 00000100:00100000:2.0:1767320441.798803:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320441.798854:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070464 00000100:00100000:3.0:1767320441.798857:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070464 00000100:00100000:3.0:1767320441.798878:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2271 00000100:00100000:3.0:1767320441.798881:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983070464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320441.798890:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320441.798910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983070464:12345-0@lo:38:kworker.0 Request processed in 28us (106us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320441.798914:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2271 00000100:00080000:2.0:1767320441.798959:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320441.798962:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320441.798964:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983070464:0@lo:38:kworker.0 00010000:00010000:2.0:1767320441.988582:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320442.310028:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320442.310056:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983070592:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320442.310151:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320442.310292:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070592 00000100:00100000:1.0:1767320442.310303:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070592 00000100:00100000:1.0:1767320442.310374:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1150 00000100:00100000:1.0:1767320442.310384:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983070592:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320442.310414:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320442.310470:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983070592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320442.310488:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1150 00000100:00100000:2.0:1767320442.310852:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983070592:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320442.311005:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983070720:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320442.311046:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320442.311240:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070720 00000100:00100000:1.0:1767320442.311246:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070720 00000100:00100000:1.0:1767320442.311299:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151 00000100:00100000:1.0:1767320442.311308:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983070720:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320442.311332:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320442.311380:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983070720:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320442.311386:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151 00000100:00100000:0.0:1767320442.311497:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983070720:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320443.014705:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320444.036785:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320444.812700:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320444.812744:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320444.813108:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681920 00000100:00100000:3.0:1767320444.813121:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681920 00000100:00100000:3.0:1767320444.813209:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915681792 00000100:00100000:3.0:1767320444.813213:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915681792 00000100:00100000:3.0:1767320444.813231:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1718 00000100:00100000:3.0:1767320444.813239:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915681920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320444.813364:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915681920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 125us (675us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320444.813373:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1718 00000100:00100000:3.0:1767320444.813390:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2272 00000100:00100000:3.0:1767320444.813395:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915681792:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320444.813415:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320444.813441:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915681792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (696us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320444.813447:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2272 00000100:00080000:1.0:1767320444.813628:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320445.060771:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320446.084659:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320446.917439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320446.917449:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320446.917518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320446.917522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320446.917526:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320446.917533:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5690 00000100:00080000:3.0:1767320446.917537:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320446.917543:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320446.917571:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c000 x1853163983070976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320446.917743:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320446.917758:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983070848:0@lo:400:kworker.0 00000100:00100000:0.0:1767320446.917819:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320446.917925:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070848 00000100:00100000:2.0:1767320446.917935:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070848 00000100:00100000:2.0:1767320446.918004:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1719 00000100:00080000:1.0:1767320446.918012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320446.918015:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983070848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320446.918075:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983070848:12345-0@lo:400:kworker.0 Request processed in 62us (256us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320446.918082:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1719 00010000:00080000:1.0:1767320446.918141:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f800 x1853163983070848/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320462 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320446.918155:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983070848:0@lo:400:kworker.0 00000100:00080000:3.0:1767320446.918171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320446.918175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320446.918177:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320446.918181:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5690 00000100:00080000:3.0:1767320446.918183:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320446.918187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320446.918194:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320446.918199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f100 x1853163983071232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320446.918200:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983071104:0@lo:400:kworker.0 00000100:00080000:3.0:1767320446.918208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320446.918211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320446.918213:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320446.918216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5690 00000100:00080000:3.0:1767320446.918218:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320446.918222:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320446.918232:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543dc00 x1853163983071360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320446.918239:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320446.918258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320446.918261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320446.918263:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320446.918293:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983071488:0@lo:400:kworker.0 00000100:00080000:3.0:1767320446.918297:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5690 00000100:00080000:3.0:1767320446.918300:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320446.918303:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320446.918321:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320446.918351:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071488 00000100:00100000:1.0:1767320446.918356:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071488 00000100:00100000:1.0:1767320446.918391:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059 00000100:00100000:1.0:1767320446.918397:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983071488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.918432:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983071488:12345-0@lo:400:kworker.0 Request processed in 36us (110us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320446.918438:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059 00000100:00080000:1.0:1767320446.918454:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320446.918461:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983071616:0@lo:400:kworker.0 00000100:00100000:2.0:1767320446.918462:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983070976:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.918483:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320446.918484:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320446.918498:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983071232:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.918507:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071104 00000100:00100000:1.0:1767320446.918510:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071104 00000100:00100000:2.0:1767320446.918517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320446.918532:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983071360:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.918534:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983070976 00000100:00100000:1.0:1767320446.918537:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983070976 00000100:00100000:1.0:1767320446.918545:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071232 00000100:00100000:1.0:1767320446.918547:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071232 00000100:00100000:1.0:1767320446.918555:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720 00000100:00100000:1.0:1767320446.918561:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983071104:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320446.918565:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320446.918589:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983071104:12345-0@lo:400:kworker.0 Request processed in 28us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320446.918595:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720 00000100:00100000:2.0:1767320446.918604:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2273 00000100:00100000:2.0:1767320446.918609:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983070976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320446.918630:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320446.918661:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983070976:12345-0@lo:38:kworker.0 Request processed in 51us (176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320446.918770:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543e300 x1853163983071744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320446.918798:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071360 00000100:00100000:3.0:1767320446.918802:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071360 00000100:00100000:3.0:1767320446.918822:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 593 00000100:00100000:3.0:1767320446.918827:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983071360:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320446.918837:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320446.918862:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983071360:12345-0@lo:38:kworker.0 Request processed in 34us (296us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320446.918867:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 593 00000100:00100000:3.0:1767320446.918885:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071616 00000100:00100000:3.0:1767320446.918888:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071616 00000100:00100000:3.0:1767320446.918908:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060 00000100:00100000:3.0:1767320446.918913:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983071616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320446.918940:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983071616:12345-0@lo:400:kworker.0 Request processed in 26us (456us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320446.918946:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060 00000100:00100000:1.0:1767320446.918985:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2274 00000100:00100000:1.0:1767320446.918991:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983071232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320446.919001:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320446.919026:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983071232:12345-0@lo:38:kworker.0 Request processed in 35us (508us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320446.919032:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2274 00000100:00080000:1.0:1767320446.919046:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320446.919049:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320446.919073:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983071616:0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.919094:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983071488:0@lo:400:kworker.0 00010000:00080000:1.0:1767320446.919104:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543c380 x1853163983071104/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320462 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320446.919112:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983071104:0@lo:400:kworker.0 00000100:00100000:1.0:1767320446.919148:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983071744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320446.919171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320446.919193:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.919197:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.919200:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983070976:0@lo:38:kworker.0 00000100:00080000:1.0:1767320446.919208:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.919212:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.919226:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983071232:0@lo:38:kworker.0 00000100:00080000:1.0:1767320446.919236:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.919238:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.919241:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983071360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320446.919295:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071744 00000100:00100000:0.0:1767320446.919300:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071744 00000100:00100000:0.0:1767320446.919335:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2275 00000100:00100000:0.0:1767320446.919340:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983071744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320446.919349:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320446.921142:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2273 00000100:00100000:0.0:1767320446.929177:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983071744:12345-0@lo:38:kworker.0 Request processed in 9825us (9994us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320446.929193:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2275 00000100:00080000:1.0:1767320446.929904:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320446.929912:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320446.929916:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983071744:0@lo:38:kworker.0 00010000:00010000:1.0:1767320447.108489:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320447.430512:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320447.430539:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983071872:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320447.430616:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320447.430722:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983071872 00000100:00100000:0.0:1767320447.430732:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983071872 00000100:00100000:0.0:1767320447.430795:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152 00000100:00100000:0.0:1767320447.430805:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983071872:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320447.430840:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320447.430903:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983071872:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320447.430910:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152 00000100:00100000:0.0:1767320447.430955:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983071872:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320447.431257:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320447.431268:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983072000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320447.431317:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320447.431474:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072000 00000100:00100000:2.0:1767320447.431480:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072000 00000100:00100000:2.0:1767320447.431515:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1153 00000100:00100000:2.0:1767320447.431523:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983072000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320447.431540:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320447.431586:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983072000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320447.431592:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1153 00000100:00100000:1.0:1767320447.431698:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983072000:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320448.133979:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320449.157714:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320449.931660:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320449.931852:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682176 00000100:00100000:1.0:1767320449.931867:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682176 00000100:00100000:1.0:1767320449.931966:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1721 00000100:00100000:1.0:1767320449.931977:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915682176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320449.932102:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915682176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 126us (458us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320449.932110:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1721 00000100:00080000:2.0:1767320449.932234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320449.933393:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320449.933481:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682048 00000100:00100000:1.0:1767320449.933491:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682048 00000100:00100000:1.0:1767320449.933540:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2276 00000100:00100000:1.0:1767320449.933547:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915682048:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320449.933569:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320449.933798:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915682048:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 249us (413us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320449.933806:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2276 00010000:00010000:1.0:1767320450.181201:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320451.204605:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320452.036482:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320452.036495:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320452.036574:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320452.036579:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320452.036584:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320452.036595:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5695 00000100:00080000:2.0:1767320452.036602:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320452.036613:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320452.036652:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b480 x1853163983072256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320452.036708:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320452.036710:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983072128:0@lo:400:kworker.0 00000100:00080000:2.0:1767320452.036713:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320452.036726:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320452.036732:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5695 00000100:00080000:2.0:1767320452.036736:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320452.036742:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320452.036760:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163983072512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320452.036773:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320452.036778:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320452.036781:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320452.036785:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5695 00000100:00080000:2.0:1767320452.036788:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320452.036794:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320452.036802:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.036812:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435ad80 x1853163983072640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320452.036848:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320452.036853:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320452.036855:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320452.036859:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320452.036861:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5695 00000100:00080000:2.0:1767320452.036867:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320452.036877:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320452.036878:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983072896:0@lo:400:kworker.0 00000100:00100000:2.0:1767320452.036895:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163983073024/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320452.036927:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.036941:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983072384:0@lo:400:kworker.0 00000100:00100000:1.0:1767320452.036986:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072896 00000100:00100000:1.0:1767320452.036997:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072896 00000100:00100000:2.0:1767320452.037024:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.037084:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320452.037092:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983072768:0@lo:400:kworker.0 00000100:00100000:1.0:1767320452.037102:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061 00000100:00100000:1.0:1767320452.037127:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983072896:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320452.037135:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320452.037525:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072128 00000100:00100000:0.0:1767320452.037536:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072128 00000100:00100000:0.0:1767320452.037591:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072384 00000100:00100000:0.0:1767320452.037594:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072384 00000100:00100000:0.0:1767320452.037623:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1722 00000100:00100000:0.0:1767320452.037633:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983072128:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320452.037715:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983072128:12345-0@lo:400:kworker.0 Request processed in 81us (911us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320452.037728:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1722 00000100:00100000:1.0:1767320452.037731:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983072896:12345-0@lo:400:kworker.0 Request processed in 602us (799us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320452.037749:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061 00000100:00100000:0.0:1767320452.037755:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1723 00000100:00100000:0.0:1767320452.037766:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983072384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320452.037811:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983072384:12345-0@lo:400:kworker.0 Request processed in 48us (789us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320452.037816:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1723 00000100:00100000:0.0:1767320452.037885:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983072896:0@lo:400:kworker.0 00010000:00080000:0.0:1767320452.037900:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b800 x1853163983072128/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320468 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320452.037911:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983072128:0@lo:400:kworker.0 00000100:00080000:0.0:1767320452.037952:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320452.037958:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:2.0:1767320452.038316:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435a300 x1853163983072384/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320468 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320452.038328:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983072384:0@lo:400:kworker.0 00000100:00100000:2.0:1767320452.038370:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983072256:0@lo:38:kworker.0 00000100:00100000:2.0:1767320452.038400:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.038416:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983072512:0@lo:38:kworker.0 00000100:00100000:2.0:1767320452.038433:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.038445:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983072640:0@lo:38:kworker.0 00000100:00100000:2.0:1767320452.038464:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.038497:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983073024:0@lo:38:kworker.0 00000100:00100000:2.0:1767320452.038517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320452.038538:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072256 00000100:00100000:2.0:1767320452.038542:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072256 00000100:00100000:2.0:1767320452.038564:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072512 00000100:00100000:2.0:1767320452.038567:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072512 00000100:00100000:2.0:1767320452.038575:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073024 00000100:00100000:2.0:1767320452.038577:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073024 00000100:00100000:2.0:1767320452.038589:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2277 00000100:00100000:2.0:1767320452.038594:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983072256:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320452.038616:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320452.038652:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983072256:12345-0@lo:38:kworker.0 Request processed in 56us (250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320452.038659:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2277 00000100:00100000:2.0:1767320452.038671:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2278 00000100:00100000:2.0:1767320452.038675:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983072512:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320452.038683:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320452.038705:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983072512:12345-0@lo:38:kworker.0 Request processed in 30us (271us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320452.038711:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2278 00000100:00100000:2.0:1767320452.038735:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2279 00000100:00100000:2.0:1767320452.038740:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983073024:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320452.038748:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320452.038770:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983073024:12345-0@lo:38:kworker.0 Request processed in 29us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320452.038775:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2279 00000100:00080000:2.0:1767320452.038796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320452.038801:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320452.038804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983072256:0@lo:38:kworker.0 00000100:00080000:2.0:1767320452.038812:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320452.038815:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320452.038817:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983072512:0@lo:38:kworker.0 00000100:00080000:2.0:1767320452.038824:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320452.038827:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320452.038830:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983073024:0@lo:38:kworker.0 00000100:00100000:2.0:1767320452.038876:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072640 00000100:00100000:2.0:1767320452.038880:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072640 00000100:00100000:2.0:1767320452.038902:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 594 00000100:00100000:2.0:1767320452.038907:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983072640:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320452.038916:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320452.038944:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983072640:12345-0@lo:38:kworker.0 Request processed in 36us (478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320452.038950:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 594 00000100:00080000:2.0:1767320452.038982:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320452.038986:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320452.038989:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983072640:0@lo:38:kworker.0 00000100:00100000:3.0:1767320452.042402:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983072768 00000100:00100000:3.0:1767320452.042420:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983072768 00000100:00080000:2.0:1767320452.042559:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320452.042659:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062 00000100:00100000:3.0:1767320452.042673:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983072768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320452.044587:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983072768:12345-0@lo:400:kworker.0 Request processed in 1905us (7444us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320452.044604:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062 00000100:00100000:2.0:1767320452.047556:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983072768:0@lo:400:kworker.0 00010000:00010000:2.0:1767320452.232509:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320452.549332:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983073152:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320452.549435:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320452.549602:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073152 00000100:00100000:1.0:1767320452.549625:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073152 00000100:00100000:1.0:1767320452.549699:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1154 00000100:00100000:1.0:1767320452.549710:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983073152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320452.549751:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320452.549810:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983073152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320452.549818:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1154 00000100:00100000:1.0:1767320452.549880:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983073152:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320452.550211:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983073280:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320452.550246:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320452.550347:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073280 00000100:00100000:0.0:1767320452.550352:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073280 00000100:00100000:0.0:1767320452.550390:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1155 00000100:00100000:0.0:1767320452.550397:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983073280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320452.550419:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320452.550468:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983073280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (221us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320452.550475:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1155 00000100:00100000:2.0:1767320452.550650:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983073280:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320453.257331:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320454.278969:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320455.051780:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320455.051897:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682432 00000100:00100000:2.0:1767320455.051931:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682432 00000100:00100000:2.0:1767320455.052371:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1724 00000100:00100000:2.0:1767320455.052386:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915682432:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320455.052527:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915682432:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 144us (759us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320455.052537:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1724 00000100:00080000:0.0:1767320455.056115:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320455.061637:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320455.061760:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682304 00000100:00100000:1.0:1767320455.061773:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682304 00000100:00100000:1.0:1767320455.061889:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2280 00000100:00100000:1.0:1767320455.061899:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915682304:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320455.061939:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320455.062191:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915682304:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 291us (568us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320455.062200:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2280 00010000:00010000:2.0:1767320455.301885:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320456.326630:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320457.158051:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320457.158067:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767320457.158218:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983073408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320457.158368:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320457.158493:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073408 00000100:00100000:1.0:1767320457.158505:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073408 00000100:00100000:1.0:1767320457.158594:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1725 00000100:00100000:1.0:1767320457.158606:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983073408:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320457.158695:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320457.158700:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320457.158704:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320457.158710:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5700 00000100:00080000:2.0:1767320457.158715:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320457.158720:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320457.158748:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323800 x1853163983073536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320457.158784:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320457.158788:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320457.158789:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320457.158793:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5700 00000100:00080000:2.0:1767320457.158795:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320457.158799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320457.158809:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163983073792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320457.158817:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320457.158821:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320457.158822:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320457.158825:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5700 00000100:00080000:2.0:1767320457.158828:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320457.158831:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320457.158841:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163983073920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320457.158914:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983073536:0@lo:38:kworker.0 00000100:00100000:0.0:1767320457.159110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320457.159140:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983073792:0@lo:38:kworker.0 00000100:00100000:0.0:1767320457.159160:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320457.159175:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983073920:0@lo:38:kworker.0 00000100:00100000:0.0:1767320457.159196:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320457.159246:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320457.159266:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983074048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.159312:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320457.159338:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320457.159344:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983073664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.159366:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320457.159406:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073920 00000100:00100000:0.0:1767320457.159412:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073920 00000100:00100000:0.0:1767320457.159439:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 595 00000100:00100000:0.0:1767320457.159445:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983073920:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320457.159469:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320457.159507:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983073920:12345-0@lo:38:kworker.0 Request processed in 61us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320457.159514:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 595 00000100:00100000:0.0:1767320457.159537:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073536 00000100:00100000:0.0:1767320457.159541:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073536 00000100:00100000:0.0:1767320457.159564:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073792 00000100:00100000:0.0:1767320457.159567:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073792 00000100:00100000:0.0:1767320457.159573:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983073664 00000100:00100000:0.0:1767320457.159575:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983073664 00000100:00100000:0.0:1767320457.159600:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1726 00000100:00100000:0.0:1767320457.159606:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983073664:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.159643:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983073664:12345-0@lo:400:kworker.0 Request processed in 36us (276us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320457.159649:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1726 00000100:00100000:0.0:1767320457.159662:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2281 00000100:00100000:0.0:1767320457.159666:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983073536:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320457.159676:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320457.159700:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983073536:12345-0@lo:38:kworker.0 Request processed in 32us (589us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320457.159706:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2281 00000100:00100000:0.0:1767320457.159716:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2282 00000100:00100000:0.0:1767320457.159720:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983073792:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320457.159728:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320457.159750:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983073792:12345-0@lo:38:kworker.0 Request processed in 29us (589us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320457.159756:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2282 00000100:00100000:0.0:1767320457.159778:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983074176:0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.159805:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320457.159834:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074048 00000100:00100000:0.0:1767320457.159837:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074048 00000100:00100000:0.0:1767320457.159865:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074176 00000100:00100000:0.0:1767320457.159868:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074176 00000100:00100000:0.0:1767320457.159889:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1063 00000100:00100000:0.0:1767320457.159896:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983074048:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.159926:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983074048:12345-0@lo:400:kworker.0 Request processed in 31us (614us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320457.163391:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1063 00000100:00100000:0.0:1767320457.163447:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064 00000100:00100000:0.0:1767320457.163472:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983074176:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.163632:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983074176:12345-0@lo:400:kworker.0 Request processed in 175us (3825us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320457.163639:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064 00000100:00080000:0.0:1767320457.163673:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320457.163679:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320457.163684:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320457.163734:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983074176:0@lo:400:kworker.0 00010000:00080000:0.0:1767320457.163785:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604323100 x1853163983073664/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320473 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320457.163799:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983073664:0@lo:400:kworker.0 00000100:00100000:0.0:1767320457.163814:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983074048:0@lo:400:kworker.0 00000100:00080000:0.0:1767320457.163860:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320457.163867:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320457.163870:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983073536:0@lo:38:kworker.0 00000100:00080000:0.0:1767320457.163883:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320457.163886:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320457.163891:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983073792:0@lo:38:kworker.0 00000100:00080000:0.0:1767320457.163901:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320457.163905:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320457.163908:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983073920:0@lo:38:kworker.0 00000100:00100000:1.0:1767320457.164479:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983073408:12345-0@lo:400:kworker.0 Request processed in 5871us (6106us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320457.164490:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1725 00010000:00080000:3.0:1767320457.164676:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320700 x1853163983073408/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320473 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320457.164690:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983073408:0@lo:400:kworker.0 00000100:00080000:2.0:1767320457.164777:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320457.164784:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320457.164788:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320457.164793:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5700 00000100:00080000:2.0:1767320457.164797:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320457.164804:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320457.164845:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604322a00 x1853163983074304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320457.165109:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983074304:0@lo:38:kworker.0 00000100:00100000:0.0:1767320457.165176:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320457.169734:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074304 00000100:00100000:1.0:1767320457.169759:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074304 00000100:00100000:1.0:1767320457.170063:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2283 00000100:00100000:1.0:1767320457.170074:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983074304:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320457.170106:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320457.170239:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983074304:12345-0@lo:38:kworker.0 Request processed in 164us (5061us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320457.170247:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2283 00000100:00080000:2.0:1767320457.170357:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320457.170365:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320457.170370:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983074304:0@lo:38:kworker.0 00010000:00010000:2.0:1767320457.355026:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320457.668998:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320457.669034:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983074432:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320457.669051:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320457.669064:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983074560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320457.670526:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320457.670913:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074560 00000100:00100000:1.0:1767320457.670934:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074560 00000100:00100000:1.0:1767320457.671240:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1156 00000100:00100000:1.0:1767320457.671252:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983074560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320457.671313:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320457.671398:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983074560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 148us (880us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320457.671406:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1156 00000100:00100000:2.0:1767320457.671495:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983074560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320457.671817:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320457.671944:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074432 00000100:00100000:1.0:1767320457.671950:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074432 00000100:00100000:1.0:1767320457.672789:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157 00000100:00100000:1.0:1767320457.672800:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983074432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320457.672827:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320457.672898:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983074432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (1081us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320457.672909:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157 00000100:00100000:0.0:1767320457.672995:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983074432:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320458.372596:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320459.398039:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320460.175651:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320460.175724:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320460.175887:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682688 00000100:00100000:1.0:1767320460.175902:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682688 00000100:00100000:0.0:1767320460.175936:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682560 00000100:00100000:0.0:1767320460.175951:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682560 00000100:00080000:2.0:1767320460.176047:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320460.176074:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1727 00000100:00100000:0.0:1767320460.176097:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915682688:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320460.176246:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915682688:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 159us (607us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320460.178942:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2284 00000100:00100000:1.0:1767320460.178960:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915682560:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320460.179000:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320460.179066:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915682560:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 111us (3341us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320460.179075:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2284 00000100:00100000:0.0:1767320460.181870:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1727 00010000:00010000:2.0:1767320460.420852:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320461.444569:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320462.276366:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320462.276377:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320462.276503:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320462.276507:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320462.276510:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320462.276519:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5705 00000100:00080000:2.0:1767320462.276524:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320462.276532:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320462.276544:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983074688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320462.276557:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321880 x1853163983074816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320462.276602:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320462.276605:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320462.276606:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320462.276611:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5705 00000100:00080000:2.0:1767320462.276613:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320462.276627:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320462.276640:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163983075072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320462.276648:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320462.276651:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320462.276653:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320462.276656:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5705 00000100:00080000:2.0:1767320462.276659:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320462.276662:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320462.276673:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320000 x1853163983075200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320462.276691:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320462.276694:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320462.276696:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320462.276699:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5705 00000100:00080000:2.0:1767320462.276701:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320462.276704:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320462.276712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323100 x1853163983075584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320462.276746:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983075328:0@lo:400:kworker.0 00000100:00100000:2.0:1767320462.276813:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320462.276885:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320462.276891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983074944:0@lo:400:kworker.0 00000100:00100000:2.0:1767320462.276928:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320462.277047:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075328 00000100:00100000:0.0:1767320462.277056:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075328 00000100:00100000:1.0:1767320462.277083:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983075456:0@lo:400:kworker.0 00000100:00100000:1.0:1767320462.277116:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320462.277164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320462.277245:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075456 00000100:00100000:0.0:1767320462.277249:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075456 00000100:00100000:1.0:1767320462.277253:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065 00000100:00100000:1.0:1767320462.277265:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983075328:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320462.277308:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1066 00000100:00100000:0.0:1767320462.277317:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983075456:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320462.277349:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983075328:12345-0@lo:400:kworker.0 Request processed in 86us (536us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320462.277358:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065 00000100:00100000:0.0:1767320462.277372:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983075456:12345-0@lo:400:kworker.0 Request processed in 56us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320462.277383:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1066 00000100:00080000:0.0:1767320462.277409:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320462.277421:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983074816:0@lo:38:kworker.0 00000100:00100000:1.0:1767320462.277423:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983075328:0@lo:400:kworker.0 00000100:00100000:0.0:1767320462.277455:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320462.277468:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983075456:0@lo:400:kworker.0 00000100:00100000:0.0:1767320462.277490:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983075072:0@lo:38:kworker.0 00000100:00100000:1.0:1767320462.277512:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074944 00000100:00100000:0.0:1767320462.277518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320462.277518:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074944 00000100:00100000:0.0:1767320462.277538:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983075200:0@lo:38:kworker.0 00000100:00100000:1.0:1767320462.277569:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074816 00000100:00100000:1.0:1767320462.277573:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074816 00000100:00100000:0.0:1767320462.277579:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320462.277595:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075072 00000100:00100000:1.0:1767320462.277599:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075072 00000100:00100000:0.0:1767320462.277618:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983075584:0@lo:38:kworker.0 00000100:00100000:1.0:1767320462.277625:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728 00000100:00100000:1.0:1767320462.277635:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983074944:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320462.277643:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320462.277676:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075200 00000100:00100000:1.0:1767320462.277680:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983074944:12345-0@lo:400:kworker.0 Request processed in 46us (749us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320462.277681:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075200 00000100:00100000:1.0:1767320462.277689:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728 00000100:00100000:1.0:1767320462.277701:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075584 00000100:00100000:1.0:1767320462.277704:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075584 00000100:00100000:1.0:1767320462.277715:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2285 00000100:00100000:1.0:1767320462.277719:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983074816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320462.277736:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320462.277760:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983074816:12345-0@lo:38:kworker.0 Request processed in 41us (306us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320462.277767:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2285 00000100:00100000:1.0:1767320462.277775:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2286 00000100:00100000:1.0:1767320462.277779:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983075072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320462.277787:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320462.277805:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983075072:12345-0@lo:38:kworker.0 Request processed in 25us (290us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320462.277811:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2286 00000100:00100000:1.0:1767320462.277819:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2287 00000100:00100000:1.0:1767320462.277823:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983075584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320462.277830:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320462.277857:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983075584:12345-0@lo:38:kworker.0 Request processed in 32us (214us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320462.277862:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2287 00000100:00080000:1.0:1767320462.277879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320462.277894:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320700 x1853163983074944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320478 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320462.277904:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983074944:0@lo:400:kworker.0 00000100:00100000:1.0:1767320462.277938:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 596 00000100:00100000:1.0:1767320462.277943:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983075200:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320462.277952:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320462.277973:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983075200:12345-0@lo:38:kworker.0 Request processed in 29us (395us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320462.277979:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 596 00000100:00080000:1.0:1767320462.278004:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320462.278008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320462.278011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983074816:0@lo:38:kworker.0 00000100:00080000:1.0:1767320462.278018:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320462.278021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320462.278024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983075072:0@lo:38:kworker.0 00000100:00080000:1.0:1767320462.278030:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320462.278032:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320462.278035:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983075200:0@lo:38:kworker.0 00000100:00080000:1.0:1767320462.278041:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320462.278044:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320462.278047:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983075584:0@lo:38:kworker.0 00000100:00100000:3.0:1767320462.279239:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320462.279908:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983074688 00000100:00100000:1.0:1767320462.279919:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983074688 00000100:00100000:1.0:1767320462.280009:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1729 00000100:00100000:1.0:1767320462.280017:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983074688:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320462.280086:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983074688:12345-0@lo:400:kworker.0 Request processed in 71us (1000us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320462.280094:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1729 00000100:00080000:0.0:1767320462.280204:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320462.280224:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322a00 x1853163983074688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320478 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320462.280239:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983074688:0@lo:400:kworker.0 00010000:00010000:1.0:1767320462.468429:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320462.788970:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320462.788998:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983075712:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320462.789070:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320462.789136:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075712 00000100:00100000:0.0:1767320462.789155:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075712 00000100:00100000:0.0:1767320462.789211:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1158 00000100:00100000:0.0:1767320462.789220:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983075712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320462.789310:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320462.789364:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983075712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 145us (293us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320462.789371:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1158 00000100:00100000:2.0:1767320462.789515:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983075712:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320462.789946:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983075840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320462.789985:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320462.790060:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983075840 00000100:00100000:1.0:1767320462.790065:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983075840 00000100:00100000:1.0:1767320462.790130:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1159 00000100:00100000:1.0:1767320462.790138:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983075840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320462.790190:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320462.790230:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983075840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320462.790236:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1159 00000100:00100000:0.0:1767320462.791772:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983075840:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320463.492579:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320464.274489:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767320464.274532:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915682816 offset 0 length 520: 4 00000100:00100000:0.0:1767320464.274558:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320464.274645:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683072 00000100:00100000:2.0:1767320464.274656:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683072 00000100:00100000:1.0:1767320464.274930:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915682944 00000100:00100000:1.0:1767320464.274935:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915682944 00000100:00100000:1.0:1767320464.274991:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2288 00000100:00100000:1.0:1767320464.275000:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915682944:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320464.275026:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320464.275105:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915682944:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 104us (545us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320464.275111:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2288 00000100:00100000:2.0:1767320464.276678:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1730 00000100:00100000:2.0:1767320464.276693:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915683072:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320464.276763:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915683072:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (2285us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320464.276770:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1730 00000100:00080000:0.0:1767320464.277249:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320464.516500:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320465.540488:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320466.372414:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320441/real 1767320441] req@ffff934610f53480 x1853163983069440/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320466 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320466.372502:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934610f53480 x1853163983069440/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320466 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320466.372537:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320466.372563:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320466.372567:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983069440:0@lo:250:kworker.0 00000100:00080000:2.0:1767320466.376885:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320466.376898:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320466.376904:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320466.376914:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5690 00000100:00080000:2.0:1767320466.376919:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.376926:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320466.376966:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323100 x1853163983075968/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320466.377058:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320466.377062:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320466.377064:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320466.377068:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5710 00000100:00080000:2.0:1767320466.377071:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.377075:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320466.377090:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604321180 x1853163983076224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320466.377106:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1767320466.377124:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320466.377129:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983076096:0@lo:400:kworker.0 00000100:00080000:2.0:1767320466.377130:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320466.377134:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320466.377139:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5710 00000100:00080000:2.0:1767320466.377142:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.377146:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320466.377158:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320700 x1853163983076480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320466.377168:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320466.377171:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320466.377173:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320466.377176:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5710 00000100:00080000:2.0:1767320466.377179:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.377182:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320466.377195:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320e00 x1853163983076608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320466.377214:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320466.377223:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320466.377228:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320466.377230:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320466.377234:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5710 00000100:00080000:2.0:1767320466.377238:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320466.377241:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320466.377252:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35f80 x1853163983076992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320466.377311:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983076352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.377322:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983076864:0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.377348:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320466.377360:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320466.377369:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320466.377377:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983076736:0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.377403:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320466.377406:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076096 00000100:00100000:0.0:1767320466.377415:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076096 00000100:00100000:2.0:1767320466.377448:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076864 00000100:00100000:2.0:1767320466.377453:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076864 00000100:00100000:0.0:1767320466.377470:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076352 00000100:00100000:0.0:1767320466.377475:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076352 00000100:00100000:2.0:1767320466.377489:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076736 00000100:00100000:2.0:1767320466.377495:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076736 00000100:00100000:0.0:1767320466.377509:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1731 00000100:00100000:0.0:1767320466.377522:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983076096:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.377522:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067 00000100:00100000:2.0:1767320466.377534:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983076864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320466.377635:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068 00000100:00100000:1.0:1767320466.377645:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983076736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320466.377651:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732 00000100:00100000:3.0:1767320466.377663:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983076352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.377692:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983076096:12345-0@lo:400:kworker.0 Request processed in 173us (479us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320466.377699:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1731 00000100:00080000:0.0:1767320466.377754:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320466.377758:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320466.377760:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320466.377797:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983076864:0@lo:400:kworker.0 00010000:00080000:0.0:1767320466.377810:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320000 x1853163983076096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320482 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320466.377820:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983076096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320466.377858:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983075968:0@lo:250:kworker.0 00000100:00100000:1.0:1767320466.378210:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983076736:12345-0@lo:400:kworker.0 Request processed in 565us (804us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320466.378216:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068 00000100:00100000:1.0:1767320466.378245:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983076736:0@lo:400:kworker.0 00000100:00100000:3.0:1767320466.378592:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983076352:12345-0@lo:400:kworker.0 Request processed in 930us (1242us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320466.378599:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732 00010000:00080000:1.0:1767320466.378692:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604321880 x1853163983076352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320482 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320466.378702:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983076352:0@lo:400:kworker.0 00000100:00100000:2.0:1767320466.378797:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983076864:12345-0@lo:400:kworker.0 Request processed in 1266us (1437us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320466.378805:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067 00000400:00000100:0.0:1767320466.380669:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983075968 offset 0 length 520: 4 00000100:00100000:0.0:1767320466.380719:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983076224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320466.380779:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320466.380817:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983076480:0@lo:38:kworker.0 00000100:00100000:3.0:1767320466.380856:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076224 00000100:00100000:3.0:1767320466.380880:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076224 00000100:00100000:3.0:1767320466.380931:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2289 00000100:00100000:0.0:1767320466.380932:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320466.380942:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983076224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320466.380967:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320466.381009:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983076224:12345-0@lo:38:kworker.0 Request processed in 67us (229us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320466.381027:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2289 00000100:00100000:3.0:1767320466.381374:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076480 00000100:00100000:3.0:1767320466.381379:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076480 00000100:00100000:0.0:1767320466.381431:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983076608:0@lo:38:kworker.0 00000100:00100000:0.0:1767320466.381465:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320466.381486:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983076992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320466.381508:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320466.382113:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320466.382119:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320466.382123:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983076224:0@lo:38:kworker.0 00000100:00100000:0.0:1767320466.382157:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076608 00000100:00100000:0.0:1767320466.382162:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076608 00000100:00100000:0.0:1767320466.383976:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 597 00000100:00100000:0.0:1767320466.384424:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983076608:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320466.384455:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320466.384563:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983076608:12345-0@lo:38:kworker.0 Request processed in 140us (3096us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320466.384571:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 597 00000100:00080000:0.0:1767320466.384675:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320466.384681:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320466.384685:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983076608:0@lo:38:kworker.0 00000100:00100000:2.0:1767320466.390685:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983076992 00000100:00100000:2.0:1767320466.390701:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983076992 00000100:00100000:1.0:1767320466.390725:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2290 00000100:00100000:1.0:1767320466.390737:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983076480:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320466.390749:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2291 00000100:00100000:2.0:1767320466.390756:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983076992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320466.390773:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:2.0:1767320466.390781:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320466.390887:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983076480:12345-0@lo:38:kworker.0 Request processed in 149us (9957us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320466.390895:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2290 00000100:00100000:2.0:1767320466.391608:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983076992:12345-0@lo:38:kworker.0 Request processed in 848us (10094us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320466.391616:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2291 00000100:00080000:0.0:1767320466.391971:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320466.391977:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320466.392182:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983076480:0@lo:38:kworker.0 00000100:00080000:0.0:1767320466.392202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320466.392205:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320466.392208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983076992:0@lo:38:kworker.0 00010000:00010000:2.0:1767320466.564538:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320467.593439:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320467.908890:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983077120:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320467.909020:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320467.909409:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077120 00000100:00100000:1.0:1767320467.909418:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077120 00000100:00100000:1.0:1767320467.909947:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160 00000100:00100000:1.0:1767320467.909962:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983077120:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320467.909995:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320467.910062:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983077120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (1041us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320467.910190:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983077248:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320467.910225:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320467.910292:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077248 00000100:00100000:0.0:1767320467.910297:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077248 00000100:00100000:3.0:1767320467.910386:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983077120:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320467.911908:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160 00000100:00100000:0.0:1767320467.914479:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1161 00000100:00100000:0.0:1767320467.914505:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983077248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320467.914544:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320467.914650:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983077248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 157us (4423us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320467.914658:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1161 00000100:00100000:3.0:1767320467.914901:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983077248:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320468.612660:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320469.388348:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320469.388400:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320469.388504:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683328 00000100:00100000:2.0:1767320469.388513:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683328 00000100:00100000:2.0:1767320469.388594:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683200 00000100:00100000:2.0:1767320469.388598:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683200 00000100:00100000:2.0:1767320469.388619:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1733 00000100:00100000:2.0:1767320469.388631:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915683328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320469.388744:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915683328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (372us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320469.388751:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1733 00000100:00100000:2.0:1767320469.388770:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2292 00000100:00100000:2.0:1767320469.388774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915683200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320469.388812:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320469.388840:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915683200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 64us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320469.388846:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2292 00000100:00080000:0.0:1767320469.389080:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320469.641502:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320470.661175:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320471.492437:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320471.492460:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320471.492534:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320471.492539:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320471.492542:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320471.492551:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5714 00000100:00080000:2.0:1767320471.492556:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320471.492572:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320471.492615:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163983077504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320471.492662:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320471.492665:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320471.492676:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320471.492681:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5714 00000100:00080000:2.0:1767320471.492683:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320471.492699:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320471.492714:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34700 x1853163983077760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320471.492723:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320471.492726:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320471.492728:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320471.492740:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5714 00000100:00080000:2.0:1767320471.492743:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320471.492753:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320471.492763:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37100 x1853163983077888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320471.492770:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983077504:0@lo:38:kworker.0 00000100:00080000:2.0:1767320471.492787:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320471.492792:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320471.492793:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320471.492797:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5714 00000100:00080000:2.0:1767320471.492799:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320471.492810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320471.492821:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34380 x1853163983078272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320471.492878:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.492940:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983077760:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.492970:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.492984:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983077888:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.493007:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.493020:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983078272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320471.493040:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320471.493070:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983078144:0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.493075:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077888 00000100:00100000:1.0:1767320471.493084:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077888 00000100:00100000:1.0:1767320471.493119:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 598 00000100:00100000:3.0:1767320471.493126:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.493130:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983077888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320471.493150:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.493154:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320471.493161:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983077376:0@lo:400:kworker.0 00000100:00100000:3.0:1767320471.493196:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.493203:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983077888:12345-0@lo:38:kworker.0 Request processed in 71us (190us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320471.493211:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 598 00000100:00100000:3.0:1767320471.493227:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983077632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.493238:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077504 00000100:00100000:1.0:1767320471.493244:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077504 00000100:00100000:3.0:1767320471.493255:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.493297:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077760 00000100:00100000:1.0:1767320471.493300:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077760 00000100:00100000:1.0:1767320471.493308:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078272 00000100:00100000:1.0:1767320471.493309:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078272 00000100:00100000:3.0:1767320471.493314:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320471.493315:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077376 00000100:00100000:1.0:1767320471.493319:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077376 00000100:00100000:3.0:1767320471.493324:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983078016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.493353:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983077632 00000100:00100000:1.0:1767320471.493356:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983077632 00000100:00100000:3.0:1767320471.493359:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320471.493386:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1734 00000100:00100000:1.0:1767320471.493391:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983077376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320471.493403:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1735 00000100:00100000:3.0:1767320471.493415:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983077632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320471.493428:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983077376:12345-0@lo:400:kworker.0 Request processed in 37us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320471.493433:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1734 00000100:00100000:1.0:1767320471.493444:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2293 00000100:00100000:1.0:1767320471.493448:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983077504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320471.493456:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.493465:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983077632:12345-0@lo:400:kworker.0 Request processed in 53us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320471.493475:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1735 00000100:00100000:1.0:1767320471.493489:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983077504:12345-0@lo:38:kworker.0 Request processed in 40us (612us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320471.493494:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2293 00000100:00100000:1.0:1767320471.493503:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2294 00000100:00100000:1.0:1767320471.493506:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983077760:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320471.493510:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2295 00010000:00020000:1.0:1767320471.493514:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320471.493519:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983078272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320471.493532:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320471.493537:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983077760:12345-0@lo:38:kworker.0 Request processed in 28us (564us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320471.493543:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2294 00000100:00100000:0.0:1767320471.493638:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078144 00000100:00100000:0.0:1767320471.493644:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078144 00000100:00100000:0.0:1767320471.493685:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078016 00000100:00100000:0.0:1767320471.493688:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078016 00000100:00100000:0.0:1767320471.493728:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1069 00000100:00100000:0.0:1767320471.493736:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983078144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320471.493758:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070 00000100:00100000:2.0:1767320471.493767:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983078016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.494494:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983078144:12345-0@lo:400:kworker.0 Request processed in 756us (1367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320471.494504:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1069 00000100:00080000:0.0:1767320471.494536:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320471.494540:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320471.494542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320471.494611:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36a00 x1853163983077376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320487 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320471.494624:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983077376:0@lo:400:kworker.0 00000100:00100000:0.0:1767320471.494636:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983078144:0@lo:400:kworker.0 00010000:00080000:0.0:1767320471.494687:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36d80 x1853163983077632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320487 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320471.494694:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320471.494699:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983077632:0@lo:400:kworker.0 00000100:00080000:1.0:1767320471.494703:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.494710:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983077888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320471.494834:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983078272:12345-0@lo:38:kworker.0 Request processed in 1315us (1791us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320471.494842:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2295 00000100:00080000:1.0:1767320471.494895:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.494900:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.494904:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983078272:0@lo:38:kworker.0 00000100:00080000:1.0:1767320471.494915:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.494918:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.495004:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983077504:0@lo:38:kworker.0 00000100:00080000:1.0:1767320471.495020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320471.495024:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320471.495027:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983077760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320471.500963:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983078016:12345-0@lo:400:kworker.0 Request processed in 7188us (7595us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320471.500978:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070 00000100:00100000:0.0:1767320471.501051:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983078016:0@lo:400:kworker.0 00010000:00010000:1.0:1767320471.684851:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320472.708457:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320473.028885:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320473.028914:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983078400:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320473.028992:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320473.029190:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078400 00000100:00100000:1.0:1767320473.029198:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078400 00000100:00100000:1.0:1767320473.029295:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1162 00000100:00100000:1.0:1767320473.029306:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983078400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320473.029387:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320473.029453:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983078400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 150us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320473.029461:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1162 00000100:00080000:3.0:1767320473.029551:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320473.029591:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983078400:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320473.030347:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983078528:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320473.030386:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320473.030498:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078528 00000100:00100000:1.0:1767320473.030506:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078528 00000100:00100000:1.0:1767320473.030565:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163 00000100:00100000:1.0:1767320473.030573:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983078528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320473.030597:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320473.030668:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320473.031089:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983078528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 515us (700us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320473.031098:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163 00000100:00100000:3.0:1767320473.031298:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983078528:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320473.732503:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320474.511294:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320474.511399:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320474.511470:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683584 00000100:00100000:0.0:1767320474.511480:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683584 00000100:00100000:0.0:1767320474.511565:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683456 00000100:00100000:0.0:1767320474.511569:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683456 00000100:00100000:0.0:1767320474.511589:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1736 00000100:00100000:0.0:1767320474.511620:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915683584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320474.511728:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915683584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320474.511753:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2296 00000100:00100000:3.0:1767320474.511763:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683456:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:1.0:1767320474.511783:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767320474.511787:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320474.511820:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (422us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320474.512379:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1736 00000100:00100000:3.0:1767320474.515869:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2296 00010000:00010000:1.0:1767320474.756802:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320475.780644:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320476.612771:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320476.612782:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320476.612895:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983078656:0@lo:400:kworker.0 00000100:00100000:2.0:1767320476.612965:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320476.613087:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078656 00000100:00100000:3.0:1767320476.613094:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078656 00000100:00100000:3.0:1767320476.613149:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1737 00000100:00100000:3.0:1767320476.613156:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983078656:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320476.613219:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320476.616249:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320476.616262:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320476.616266:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320476.616290:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5720 00000100:00080000:1.0:1767320476.616295:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320476.616303:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320476.616348:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631989180 x1853163983078784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320476.616410:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320476.616414:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320476.616417:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320476.616420:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5720 00000100:00080000:1.0:1767320476.616422:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320476.616426:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320476.616439:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53480 x1853163983079040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320476.616448:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320476.616451:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320476.616453:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320476.616456:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5720 00000100:00080000:1.0:1767320476.616458:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320476.616462:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320476.616477:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163983079168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320476.616587:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983079296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.616669:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.616726:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320476.616744:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983078912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.616772:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.616809:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079296 00000100:00100000:0.0:1767320476.616817:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079296 00000100:00100000:0.0:1767320476.616874:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1767320476.616883:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983079296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.616935:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983079296:12345-0@lo:400:kworker.0 Request processed in 53us (266us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320476.616941:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1767320476.616970:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078912 00000100:00100000:0.0:1767320476.616974:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078912 00000100:00100000:0.0:1767320476.617016:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1738 00000100:00100000:0.0:1767320476.617024:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983078912:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617053:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983078912:12345-0@lo:400:kworker.0 Request processed in 31us (281us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320476.617059:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1738 00000100:00080000:0.0:1767320476.617078:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320476.617082:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:0.0:1767320476.617105:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988380 x1853163983078912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320492 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320476.617117:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983078912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617127:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983079296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617152:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079424:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617176:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.617203:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079424 00000100:00100000:0.0:1767320476.617206:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079424 00000100:00100000:0.0:1767320476.617232:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1767320476.617238:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617296:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079424:12345-0@lo:400:kworker.0 Request processed in 34us (94us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320476.617302:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072 00000100:00080000:0.0:1767320476.617315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320476.617332:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079424:0@lo:400:kworker.0 00000100:00100000:0.0:1767320476.617368:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983078784:0@lo:38:kworker.0 00000100:00100000:0.0:1767320476.617391:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.617408:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983079040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320476.617426:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.617442:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983079168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320476.617509:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320476.617550:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079168 00000100:00100000:0.0:1767320476.617553:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079168 00000100:00100000:0.0:1767320476.617592:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 599 00000100:00100000:0.0:1767320476.617598:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983079168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320476.617615:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320476.617667:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983079168:12345-0@lo:38:kworker.0 Request processed in 67us (157us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320476.617673:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 599 00000100:00100000:0.0:1767320476.617689:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983078784 00000100:00100000:0.0:1767320476.617693:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983078784 00000100:00100000:0.0:1767320476.617711:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079040 00000100:00100000:0.0:1767320476.617713:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079040 00000100:00100000:0.0:1767320476.617729:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2297 00000100:00100000:0.0:1767320476.617733:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983078784:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320476.617743:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320476.617769:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983078784:12345-0@lo:38:kworker.0 Request processed in 34us (376us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320476.617774:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2297 00000100:00100000:0.0:1767320476.617784:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2298 00000100:00100000:0.0:1767320476.617787:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983079040:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320476.617796:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320476.617817:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983079040:12345-0@lo:38:kworker.0 Request processed in 28us (391us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320476.617822:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2298 00000100:00080000:0.0:1767320476.617853:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320476.617857:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320476.617860:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983078784:0@lo:38:kworker.0 00000100:00080000:0.0:1767320476.617868:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320476.617871:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320476.617873:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983079040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320476.617880:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320476.617882:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320476.617885:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983079168:0@lo:38:kworker.0 00000100:00100000:3.0:1767320476.619284:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983078656:12345-0@lo:400:kworker.0 Request processed in 6108us (6298us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320476.619295:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1737 00000100:00080000:1.0:1767320476.621511:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320476.621521:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320476.621526:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320476.621534:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5720 00000100:00080000:1.0:1767320476.621539:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320476.621546:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320476.621598:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624000 x1853163983079552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320476.621776:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983079552:0@lo:38:kworker.0 00000100:00100000:0.0:1767320476.621843:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320476.622296:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079552 00000100:00100000:3.0:1767320476.622306:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079552 00000100:00100000:3.0:1767320476.622357:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2299 00000100:00100000:3.0:1767320476.622366:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983079552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320476.622389:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320476.622499:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983079552:12345-0@lo:38:kworker.0 Request processed in 132us (655us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320476.622506:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2299 00010000:00080000:2.0:1767320476.622769:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631989500 x1853163983078656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320492 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320476.622782:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983078656:0@lo:400:kworker.0 00000100:00080000:1.0:1767320476.623205:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320476.623213:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320476.623217:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983079552:0@lo:38:kworker.0 00010000:00010000:2.0:1767320476.804981:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320477.830459:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320478.152528:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1767320478.152563:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320478.152635:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320478.152763:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079680 00000100:00100000:0.0:1767320478.152773:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079680 00000100:00100000:1.0:1767320478.152775:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079808:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320478.152802:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320478.152840:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079808 00000100:00100000:0.0:1767320478.152844:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079808 00000100:00100000:0.0:1767320478.152864:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164 00000100:00100000:0.0:1767320478.152873:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320478.152916:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320478.152973:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 102us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320478.152981:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164 00000100:00100000:0.0:1767320478.152998:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1165 00000100:00100000:0.0:1767320478.153004:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079808:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320478.153020:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320478.153045:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983079808:12345-0@lo:13:osp-pre-0-1.0 Request processed in 41us (242us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320478.153051:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1165 00000100:00080000:2.0:1767320478.153294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320478.153300:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320478.169647:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320478.169706:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983079808:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320478.852626:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320479.628686:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320479.630720:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683840 00000100:00100000:0.0:1767320479.630734:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683840 00000100:00100000:0.0:1767320479.630889:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1739 00000100:00100000:0.0:1767320479.630900:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915683840:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320479.630998:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915683840:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (2321us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320479.631006:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1739 00000100:00100000:3.0:1767320479.636143:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320479.636320:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683712 00000100:00100000:0.0:1767320479.636333:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683712 00000100:00100000:0.0:1767320479.636393:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2300 00000100:00100000:0.0:1767320479.636403:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320479.636428:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320479.636499:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 95us (368us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320479.636507:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2300 00000100:00080000:2.0:1767320479.640757:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320479.878014:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320480.901674:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320481.732731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320481.732741:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320481.732821:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983079936:0@lo:400:kworker.0 00000100:00100000:0.0:1767320481.732881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320481.732984:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983079936 00000100:00100000:1.0:1767320481.732993:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983079936 00000100:00100000:1.0:1767320481.733061:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740 00000100:00100000:1.0:1767320481.733074:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983079936:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320481.733173:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320481.733250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320481.733254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320481.733258:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320481.733263:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5725 00000100:00080000:3.0:1767320481.733268:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320481.733299:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320481.733333:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014e00 x1853163983080064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320481.733393:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983080192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320481.733425:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320481.733454:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080192 00000100:00100000:2.0:1767320481.733459:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080192 00000100:00100000:2.0:1767320481.733501:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1741 00000100:00100000:2.0:1767320481.733508:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983080192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320481.733545:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983080192:12345-0@lo:400:kworker.0 Request processed in 36us (117us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320481.733553:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1741 00010000:00080000:2.0:1767320481.733589:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015500 x1853163983080192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320481.733603:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983080192:0@lo:400:kworker.0 00000100:00080000:2.0:1767320481.733621:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320481.733637:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983080064:0@lo:38:kworker.0 00000100:00100000:2.0:1767320481.733661:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320481.733689:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080064 00000100:00100000:2.0:1767320481.733693:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080064 00000100:00100000:2.0:1767320481.733711:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2301 00000100:00100000:2.0:1767320481.733716:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983080064:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320481.733733:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320481.733767:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983080064:12345-0@lo:38:kworker.0 Request processed in 49us (104us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320481.733773:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2301 00000100:00080000:2.0:1767320481.733796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320481.733801:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320481.733804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983080064:0@lo:38:kworker.0 00000100:00100000:1.0:1767320481.734508:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983079936:12345-0@lo:400:kworker.0 Request processed in 1434us (1621us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320481.734516:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740 00010000:00080000:0.0:1767320481.734598:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017480 x1853163983079936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320481.734611:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983079936:0@lo:400:kworker.0 00000100:00080000:3.0:1767320481.734734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320481.734739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320481.734741:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320481.734746:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5725 00000100:00080000:3.0:1767320481.734749:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320481.734754:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320481.734776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014700 x1853163983080320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320481.734835:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983080320:0@lo:38:kworker.0 00000100:00100000:2.0:1767320481.734875:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320481.734945:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080320 00000100:00100000:1.0:1767320481.734951:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080320 00000100:00100000:1.0:1767320481.734984:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2302 00000100:00100000:1.0:1767320481.734990:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983080320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320481.735005:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320481.735097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320481.735101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320481.735104:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320481.735107:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5725 00000100:00080000:3.0:1767320481.735110:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320481.735114:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320481.735129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163983080448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320481.735174:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983080448:0@lo:38:kworker.0 00000100:00100000:2.0:1767320481.735201:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320481.735258:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080448 00000100:00100000:0.0:1767320481.735263:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080448 00000100:00100000:0.0:1767320481.735300:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 600 00000100:00100000:0.0:1767320481.735305:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983080448:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320481.735327:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320481.735356:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983080448:12345-0@lo:38:kworker.0 Request processed in 49us (154us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320481.735362:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 600 00000100:00100000:1.0:1767320481.735712:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983080320:12345-0@lo:38:kworker.0 Request processed in 720us (836us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320481.735720:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2302 00000100:00080000:2.0:1767320481.735786:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320481.735791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320481.735795:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983080320:0@lo:38:kworker.0 00000100:00080000:2.0:1767320481.735813:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320481.735816:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320481.735819:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983080448:0@lo:38:kworker.0 00000100:00100000:1.0:1767320481.735979:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983080576:0@lo:400:kworker.0 00000100:00100000:1.0:1767320481.736010:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320481.736070:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080576 00000100:00100000:0.0:1767320481.736075:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080576 00000100:00100000:0.0:1767320481.736121:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073 00000100:00100000:0.0:1767320481.736129:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983080576:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320481.736189:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320481.736406:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983080704:0@lo:400:kworker.0 00000100:00100000:2.0:1767320481.736433:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320481.736509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320481.736514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320481.736516:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320481.736520:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5725 00000100:00080000:3.0:1767320481.736524:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320481.736528:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320481.736543:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163983080832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320481.736573:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983080832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320481.736602:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320481.736628:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080832 00000100:00100000:3.0:1767320481.736632:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080832 00000100:00100000:3.0:1767320481.736653:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2303 00000100:00100000:3.0:1767320481.736657:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983080832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320481.736671:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320481.736702:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983080832:12345-0@lo:38:kworker.0 Request processed in 42us (98us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320481.736707:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2303 00000100:00080000:3.0:1767320481.736733:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320481.736738:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320481.736741:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983080832:0@lo:38:kworker.0 00000100:00100000:3.0:1767320481.736765:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080704 00000100:00100000:3.0:1767320481.736768:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080704 00000100:00100000:3.0:1767320481.736796:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074 00000100:00100000:3.0:1767320481.736801:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983080704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320481.736881:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983080576:12345-0@lo:400:kworker.0 Request processed in 753us (868us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320481.736888:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073 00000100:00080000:0.0:1767320481.736917:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320481.736981:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983080576:0@lo:400:kworker.0 00000100:00100000:3.0:1767320481.737031:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983080704:12345-0@lo:400:kworker.0 Request processed in 229us (595us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320481.737037:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074 00000100:00100000:2.0:1767320481.737090:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983080704:0@lo:400:kworker.0 00010000:00010000:2.0:1767320481.926705:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320482.948491:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320483.268822:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983080960:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320483.268914:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320483.272248:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983080960 00000100:00100000:1.0:1767320483.272264:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983080960 00000100:00100000:1.0:1767320483.272412:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1166 00000100:00100000:1.0:1767320483.272422:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983080960:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320483.273160:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320483.273245:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983080960:12345-0@lo:13:osp-pre-0-1.0 Request processed in 825us (4331us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320483.273254:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1166 00000100:00100000:1.0:1767320483.273320:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081088:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320483.273360:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320483.274006:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081088 00000100:00100000:0.0:1767320483.274023:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081088 00000100:00100000:3.0:1767320483.274125:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983080960:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320483.278189:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320483.278202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320483.281857:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167 00000100:00100000:0.0:1767320483.281882:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081088:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320483.283503:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320483.283618:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1745us (10254us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320483.283627:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167 00000100:00100000:1.0:1767320483.284004:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081088:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320483.972619:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320484.748412:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320484.748526:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320484.748661:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684096 00000100:00100000:0.0:1767320484.748675:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684096 00000100:00100000:0.0:1767320484.748796:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915683968 00000100:00100000:0.0:1767320484.748800:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915683968 00000100:00100000:0.0:1767320484.748825:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1742 00000100:00100000:0.0:1767320484.748837:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915684096:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320484.748948:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915684096:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 113us (555us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320484.748957:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1742 00000100:00100000:0.0:1767320484.748980:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2304 00000100:00100000:0.0:1767320484.748985:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683968:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320484.749011:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320484.749043:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915683968:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (517us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320484.749064:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2304 00000100:00080000:2.0:1767320484.751142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320484.996605:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320486.020692:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320486.852489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320486.852507:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320486.852642:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767320486.852816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320486.852821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320486.852826:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320486.852836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320486.852842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320486.852859:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320486.852890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163983081344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320486.852970:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983081344:0@lo:38:kworker.0 00000100:00100000:0.0:1767320486.853062:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320486.857008:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081344 00000100:00100000:1.0:1767320486.857030:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081344 00000100:00100000:1.0:1767320486.857109:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2305 00000100:00100000:1.0:1767320486.857118:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983081344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320486.857144:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320486.857240:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983081344:12345-0@lo:38:kworker.0 Request processed in 123us (4177us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320486.857245:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2305 00000100:00080000:0.0:1767320486.857414:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320486.857421:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320486.857427:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983081344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320486.859515:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983081472:0@lo:400:kworker.0 00000100:00100000:2.0:1767320486.859622:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320486.859695:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983081216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320486.859918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320486.860510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320486.860520:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320486.860524:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320486.860620:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320486.860627:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320486.860635:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320486.861732:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163983081600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320486.861762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320486.861765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320486.861768:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320486.861771:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320486.861774:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320486.861778:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320486.861800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017100 x1853163983081728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320486.861835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320486.861839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320486.861841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320486.861845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5730 00000100:00080000:3.0:1767320486.861917:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320486.861923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320486.861939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014e00 x1853163983082112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320486.861991:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081984:0@lo:400:kworker.0 00000100:00100000:3.0:1767320486.862062:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320486.862109:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320486.862113:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320486.862128:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320486.862152:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081984 00000100:00100000:3.0:1767320486.862160:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081984 00000100:00100000:3.0:1767320486.862217:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081856 00000100:00100000:3.0:1767320486.862219:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081856 00000100:00100000:3.0:1767320486.862238:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1075 00000100:00100000:3.0:1767320486.862244:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081984:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320486.862305:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081984:12345-0@lo:400:kworker.0 Request processed in 62us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320486.862312:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1075 00000100:00100000:3.0:1767320486.862337:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076 00000100:00100000:3.0:1767320486.862341:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081856:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320486.862359:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983081856:12345-0@lo:400:kworker.0 Request processed in 18us (230us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320486.862362:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076 00000100:00080000:3.0:1767320486.862375:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320486.862380:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320486.862436:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081856:0@lo:400:kworker.0 00000100:00100000:3.0:1767320486.862450:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983081984:0@lo:400:kworker.0 00000100:00100000:2.0:1767320486.862512:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983081600:0@lo:38:kworker.0 00000100:00100000:2.0:1767320486.862621:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320486.864693:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983081728:0@lo:38:kworker.0 00000100:00100000:2.0:1767320486.864763:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320486.864795:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983082112:0@lo:38:kworker.0 00000100:00100000:2.0:1767320486.864818:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320486.864857:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081472 00000100:00100000:0.0:1767320486.864874:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081472 00000100:00100000:0.0:1767320486.864962:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081216 00000100:00100000:0.0:1767320486.864966:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081216 00000100:00100000:0.0:1767320486.864994:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081600 00000100:00100000:0.0:1767320486.864998:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081600 00000100:00100000:0.0:1767320486.865015:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1743 00000100:00100000:0.0:1767320486.865026:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983081472:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320486.865132:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983081472:12345-0@lo:400:kworker.0 Request processed in 107us (5509us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320486.865140:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1743 00000100:00100000:0.0:1767320486.865161:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744 00000100:00100000:0.0:1767320486.865168:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983081216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320486.865198:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983081216:12345-0@lo:400:kworker.0 Request processed in 31us (5291us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320486.865204:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744 00000100:00100000:0.0:1767320486.865217:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2306 00000100:00100000:0.0:1767320486.865222:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983081600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320486.865249:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320486.865311:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983081600:12345-0@lo:38:kworker.0 Request processed in 88us (2690us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320486.865332:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2306 00000100:00100000:0.0:1767320486.865357:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983081728 00000100:00100000:0.0:1767320486.865362:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983081728 00000100:00100000:0.0:1767320486.865381:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 601 00000100:00100000:0.0:1767320486.865396:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983081728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320486.865407:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320486.865436:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983081728:12345-0@lo:38:kworker.0 Request processed in 49us (673us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320486.865442:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 601 00000100:00080000:0.0:1767320486.865470:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320486.865525:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015180 x1853163983081472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320502 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320486.865553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983081472:0@lo:400:kworker.0 00010000:00080000:0.0:1767320486.865570:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015f80 x1853163983081216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320502 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320486.865579:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983081216:0@lo:400:kworker.0 00000100:00080000:2.0:1767320486.871484:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320486.871500:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320486.871505:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983081600:0@lo:38:kworker.0 00000100:00080000:2.0:1767320486.871520:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320486.871525:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320486.871528:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983081728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320486.871796:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082112 00000100:00100000:1.0:1767320486.871804:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082112 00000100:00100000:1.0:1767320486.871853:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2307 00000100:00100000:1.0:1767320486.871861:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983082112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320486.871881:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320486.871953:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983082112:12345-0@lo:38:kworker.0 Request processed in 91us (7132us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320486.871960:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2307 00000100:00080000:2.0:1767320486.872578:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320486.872584:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320486.872588:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983082112:0@lo:38:kworker.0 00010000:00010000:1.0:1767320487.044625:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320488.068541:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320488.388611:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320488.389184:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983082240:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320488.389371:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320488.390135:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082240 00000100:00100000:0.0:1767320488.390147:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082240 00000100:00100000:0.0:1767320488.390300:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1168 00000100:00100000:0.0:1767320488.390310:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983082240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320488.390343:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320488.390428:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983082240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (1057us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320488.390436:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1168 00000100:00100000:0.0:1767320488.390493:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983082240:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320488.390940:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983082368:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320488.391128:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320488.393807:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320488.394043:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082368 00000100:00100000:2.0:1767320488.394056:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082368 00000100:00100000:2.0:1767320488.394148:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169 00000100:00100000:2.0:1767320488.394158:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983082368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320488.394200:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767320488.394500:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320488.396703:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983082368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2531us (2892us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320488.396720:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169 00000100:00100000:0.0:1767320488.400722:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983082368:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320488.846081:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320488.846247:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684480 00000100:00100000:3.0:1767320488.846260:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684480 00000100:00100000:3.0:1767320488.846402:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1745 00000100:00100000:3.0:1767320488.846413:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915684480:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320488.846520:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915684480:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 110us (455us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320488.846529:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1745 00000100:00080000:1.0:1767320488.846903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767320488.851619:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915684224 offset 0 length 520: 4 00000100:00100000:3.0:1767320488.851682:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320488.855536:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684352 00000100:00100000:0.0:1767320488.855598:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684352 00000100:00100000:0.0:1767320488.855690:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2308 00000100:00100000:0.0:1767320488.855701:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915684352:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320488.855734:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320488.855937:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915684352:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 236us (4254us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320488.855946:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2308 00010000:00010000:2.0:1767320489.092671:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320490.116731:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320491.140751:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320491.972656:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320466/real 1767320466] req@ffff934604323100 x1853163983075968/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320491 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320491.972713:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604323100 x1853163983075968/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320491 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320491.972737:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.972746:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320491.972751:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983075968:0@lo:250:kworker.0 00000100:00080000:2.0:1767320491.972823:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320491.972830:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320491.972834:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320491.972840:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5714 00000100:00080000:2.0:1767320491.972844:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320491.972853:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320491.972908:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597800 x1853163983082496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320491.973000:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320491.973021:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320491.973025:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320491.973027:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320491.973032:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5735 00000100:00080000:2.0:1767320491.973035:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320491.973039:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320491.973058:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596300 x1853163983082752/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320491.973111:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983082496:0@lo:250:kworker.0 00000400:00000100:1.0:1767320491.973177:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983082496 offset 0 length 520: 4 00000100:00100000:1.0:1767320491.973232:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983082752:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.973264:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320491.973565:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082752 00000100:00100000:3.0:1767320491.973576:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082752 00000100:00100000:3.0:1767320491.973773:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2309 00000100:00100000:3.0:1767320491.973783:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983082752:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320491.973810:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320491.973871:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983082752:12345-0@lo:38:kworker.0 Request processed in 88us (604us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320491.973878:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2309 00000100:00080000:1.0:1767320491.974105:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.974110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.974114:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983082752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320491.975686:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983082880:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.975758:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320491.975809:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983082624:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.975834:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.975911:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082880 00000100:00100000:1.0:1767320491.975921:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082880 00000100:00100000:1.0:1767320491.976006:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983082624 00000100:00100000:1.0:1767320491.976010:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983082624 00000100:00100000:1.0:1767320491.976038:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1746 00000100:00100000:1.0:1767320491.976047:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983082880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.976111:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983082880:12345-0@lo:400:kworker.0 Request processed in 65us (351us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320491.976118:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1746 00000100:00100000:1.0:1767320491.976137:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1747 00000100:00100000:1.0:1767320491.976157:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983082624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.976193:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983082624:12345-0@lo:400:kworker.0 Request processed in 50us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320491.976199:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1747 00000100:00080000:0.0:1767320491.976326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320491.976364:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638595c00 x1853163983082880/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320507 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320491.976381:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983082880:0@lo:400:kworker.0 00010000:00080000:0.0:1767320491.976409:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597b80 x1853163983082624/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320507 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320491.976418:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983082624:0@lo:400:kworker.0 00000100:00080000:2.0:1767320491.977079:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320491.977086:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320491.977090:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320491.977096:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5735 00000100:00080000:2.0:1767320491.977101:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320491.977107:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320491.977146:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597480 x1853163983083008/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320491.977167:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320491.977171:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320491.977173:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320491.977177:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5735 00000100:00080000:2.0:1767320491.977179:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320491.977183:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320491.977197:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594a80 x1853163983083136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320491.977241:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320491.977245:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320491.977247:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320491.977251:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5735 00000100:00080000:2.0:1767320491.977254:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320491.977258:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320491.977284:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594e00 x1853163983083520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320491.977310:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983083264:0@lo:400:kworker.0 00000100:00100000:0.0:1767320491.977371:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983083392:0@lo:400:kworker.0 00000100:00100000:3.0:1767320491.977371:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320491.977442:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083264 00000100:00100000:2.0:1767320491.977448:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083264 00000100:00100000:2.0:1767320491.977502:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077 00000100:00100000:1.0:1767320491.977509:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983083008:0@lo:38:kworker.0 00000100:00100000:2.0:1767320491.977509:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983083264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.977927:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.977971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983083136:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.978002:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.978026:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983083520:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.978054:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.978090:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083136 00000100:00100000:1.0:1767320491.978094:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083136 00000100:00100000:1.0:1767320491.978121:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 602 00000100:00100000:1.0:1767320491.978126:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983083136:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.978166:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.978209:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983083136:12345-0@lo:38:kworker.0 Request processed in 81us (205us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.978217:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 602 00000100:00100000:1.0:1767320491.978245:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083008 00000100:00100000:1.0:1767320491.978249:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083008 00000100:00100000:1.0:1767320491.978320:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083520 00000100:00100000:1.0:1767320491.978324:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083520 00000100:00100000:1.0:1767320491.978344:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2310 00000100:00100000:1.0:1767320491.978349:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983083008:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.978360:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.978401:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983083008:12345-0@lo:38:kworker.0 Request processed in 38us (462us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.978409:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2310 00000100:00100000:1.0:1767320491.978422:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2311 00000100:00100000:1.0:1767320491.978426:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983083520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320491.978435:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320491.978665:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983083520:12345-0@lo:38:kworker.0 Request processed in 236us (609us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320491.978672:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2311 00000100:00080000:1.0:1767320491.978715:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.978720:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.978724:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983083008:0@lo:38:kworker.0 00000100:00080000:1.0:1767320491.978735:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.978739:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.978753:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983083136:0@lo:38:kworker.0 00000100:00080000:1.0:1767320491.978765:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320491.978769:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320491.978772:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983083520:0@lo:38:kworker.0 00000100:00100000:1.0:1767320491.978837:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320491.978892:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083392 00000100:00100000:1.0:1767320491.978897:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083392 00000100:00100000:1.0:1767320491.978932:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1078 00000100:00100000:1.0:1767320491.978940:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983083392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320491.979000:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983083392:12345-0@lo:400:kworker.0 Request processed in 61us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320491.979006:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1078 00000100:00080000:1.0:1767320491.979025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320491.979030:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320491.979054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983083392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320491.987132:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983083264:12345-0@lo:400:kworker.0 Request processed in 9604us (9741us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320491.987155:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077 00000100:00100000:3.0:1767320491.987426:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983083264:0@lo:400:kworker.0 00010000:00010000:2.0:1767320492.164693:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320493.188888:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320493.509776:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983083648:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320493.509877:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320493.509940:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320493.509955:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983083776:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320493.509979:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320493.510058:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083648 00000100:00100000:1.0:1767320493.510069:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083648 00000100:00100000:1.0:1767320493.510141:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083776 00000100:00100000:1.0:1767320493.510145:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083776 00000100:00100000:1.0:1767320493.510169:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1170 00000100:00100000:1.0:1767320493.510180:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983083648:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320493.510218:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320493.510303:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983083648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 124us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320493.510311:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1170 00000100:00100000:1.0:1767320493.510331:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1171 00000100:00100000:1.0:1767320493.510338:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983083776:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320493.510355:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320493.510383:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983083776:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320493.510389:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1171 00000100:00080000:3.0:1767320493.512523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320493.512536:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320493.512612:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983083776:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320493.512636:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983083648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320493.967231:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320493.967404:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320493.967424:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684736 00000100:00100000:1.0:1767320493.967437:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684736 00000100:00100000:1.0:1767320493.967589:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684608 00000100:00100000:1.0:1767320493.967593:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684608 00000100:00100000:1.0:1767320493.967620:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1748 00000100:00100000:1.0:1767320493.967632:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915684736:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320493.967738:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915684736:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 108us (523us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320493.967748:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1748 00000100:00100000:1.0:1767320493.967773:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2312 00000100:00100000:1.0:1767320493.967778:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915684608:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320493.967804:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320493.967835:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915684608:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (431us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320493.967842:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2312 00000100:00080000:2.0:1767320493.972314:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320494.212559:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320495.236587:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320496.260588:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320497.093323:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320497.093333:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320497.093417:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320497.093422:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320497.093425:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320497.093442:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5740 00000100:00080000:3.0:1767320497.093448:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320497.093455:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320497.093476:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163983084032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320497.093525:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320497.093528:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320497.093530:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320497.093534:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5740 00000100:00080000:3.0:1767320497.093536:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320497.093539:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320497.093552:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0a80 x1853163983084288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320497.093559:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320497.093561:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320497.093563:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320497.093565:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5740 00000100:00080000:3.0:1767320497.093567:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320497.093571:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320497.093579:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0e00 x1853163983084416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320497.093623:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320497.093627:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320497.093628:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320497.093632:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5740 00000100:00080000:3.0:1767320497.093634:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320497.093637:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320497.093647:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2300 x1853163983084800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320497.093678:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983084544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.093741:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320497.093779:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320497.093784:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983084672:0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.093804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320497.095055:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983083904:0@lo:400:kworker.0 00000100:00100000:1.0:1767320497.095165:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320497.095262:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320497.095312:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983084160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320497.095337:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320497.095386:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084544 00000100:00100000:1.0:1767320497.095397:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084544 00000100:00100000:3.0:1767320497.095420:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983083904 00000100:00100000:2.0:1767320497.095433:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084672 00000100:00100000:3.0:1767320497.095435:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983083904 00000100:00100000:2.0:1767320497.095442:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084672 00000100:00100000:1.0:1767320497.095477:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079 00000100:00100000:1.0:1767320497.095495:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983084544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.095517:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084160 00000100:00100000:3.0:1767320497.095525:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084160 00000100:00100000:0.0:1767320497.095550:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983084032:0@lo:38:kworker.0 00000100:00100000:1.0:1767320497.095591:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983084544:12345-0@lo:400:kworker.0 Request processed in 95us (1840us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320497.095593:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320497.095620:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983084288:0@lo:38:kworker.0 00000100:00100000:0.0:1767320497.095641:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320497.095652:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983084416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320497.095677:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320497.095696:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983084800:0@lo:38:kworker.0 00000100:00100000:0.0:1767320497.095720:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320497.095747:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084032 00000100:00100000:0.0:1767320497.095751:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084032 00000100:00100000:3.0:1767320497.095783:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1749 00000100:00100000:3.0:1767320497.095801:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983083904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.096077:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983083904:12345-0@lo:400:kworker.0 Request processed in 281us (911us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320497.097891:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080 00000100:00100000:2.0:1767320497.097916:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983084672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320497.098017:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983084672:12345-0@lo:400:kworker.0 Request processed in 113us (4210us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320497.098025:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080 00000100:00100000:3.0:1767320497.098366:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1749 00000100:00100000:3.0:1767320497.098408:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084288 00000100:00100000:3.0:1767320497.098416:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084288 00000100:00100000:3.0:1767320497.098436:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084800 00000100:00100000:3.0:1767320497.098438:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084800 00000100:00100000:3.0:1767320497.098446:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1750 00000100:00100000:3.0:1767320497.098454:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983084160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.098523:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983084160:12345-0@lo:400:kworker.0 Request processed in 72us (3186us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320497.098527:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1750 00000100:00100000:3.0:1767320497.098536:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2313 00000100:00100000:3.0:1767320497.098538:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320497.098554:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320497.098587:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084032:12345-0@lo:38:kworker.0 Request processed in 47us (2996us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320497.098590:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2313 00000100:00100000:3.0:1767320497.098608:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2314 00000100:00100000:3.0:1767320497.098612:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320497.098621:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320497.098657:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084288:12345-0@lo:38:kworker.0 Request processed in 43us (3014us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320497.098663:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2314 00000100:00100000:3.0:1767320497.098674:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2315 00000100:00100000:3.0:1767320497.098677:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320497.098686:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320497.098708:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983084800:12345-0@lo:38:kworker.0 Request processed in 30us (2987us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320497.098712:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2315 00010000:00080000:3.0:1767320497.098788:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015180 x1853163983084160/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320513 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320497.098799:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983084160:0@lo:400:kworker.0 00010000:00080000:3.0:1767320497.098808:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163983083904/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320513 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320497.098813:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983083904:0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.098859:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983084672:0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.098871:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983084544:0@lo:400:kworker.0 00000100:00100000:3.0:1767320497.098905:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084416 00000100:00100000:3.0:1767320497.098909:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084416 00000100:00100000:3.0:1767320497.098930:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 603 00000100:00100000:3.0:1767320497.098933:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983084416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320497.098940:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320497.098962:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983084416:12345-0@lo:38:kworker.0 Request processed in 27us (3284us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320497.098968:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 603 00000100:00080000:3.0:1767320497.099001:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320497.099007:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320497.099010:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983084032:0@lo:38:kworker.0 00000100:00080000:3.0:1767320497.099020:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320497.099023:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320497.099026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983084288:0@lo:38:kworker.0 00000100:00080000:3.0:1767320497.099035:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320497.099038:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320497.099041:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983084416:0@lo:38:kworker.0 00000100:00080000:3.0:1767320497.099049:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320497.099052:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320497.099054:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983084800:0@lo:38:kworker.0 00000100:00080000:2.0:1767320497.100635:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320497.100648:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320497.100652:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320497.104486:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079 00010000:00010000:2.0:1767320497.285302:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320498.309757:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320498.629507:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983084928:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320498.629603:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320498.629669:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983084928 00000100:00100000:0.0:1767320498.629680:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983084928 00000100:00100000:0.0:1767320498.629747:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172 00000100:00100000:0.0:1767320498.629758:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324c2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983084928:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320498.629800:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320498.629853:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324c2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983084928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (250us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320498.629862:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172 00000100:00100000:1.0:1767320498.631660:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983084928:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320498.631695:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320498.631704:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983085056:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320498.631845:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320498.634824:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085056 00000100:00100000:0.0:1767320498.634835:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085056 00000100:00100000:0.0:1767320498.634897:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173 00000100:00100000:0.0:1767320498.634911:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983085056:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320498.634949:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320498.635033:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983085056:12345-0@lo:13:osp-pre-0-1.0 Request processed in 126us (3187us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320498.635041:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173 00000100:00100000:3.0:1767320498.635496:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983085056:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320499.086511:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320499.086562:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320499.086853:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684864 00000100:00100000:0.0:1767320499.086872:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684864 00000100:00100000:3.0:1767320499.086911:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915684992 00000100:00100000:3.0:1767320499.086924:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915684992 00000100:00100000:0.0:1767320499.086963:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2316 00000100:00100000:0.0:1767320499.086974:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915684864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320499.087007:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320499.087017:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1751 00000100:00100000:3.0:1767320499.087026:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915684992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320499.087103:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915684864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 127us (600us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320499.087116:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915684992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (543us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320499.087133:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2316 00000100:00080000:1.0:1767320499.087241:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320499.089111:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1751 00010000:00010000:1.0:1767320499.334047:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320500.356544:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320501.380546:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320502.212580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320502.212593:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320502.212686:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320502.212704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320502.212708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320502.212712:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320502.212720:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5745 00000100:00080000:2.0:1767320502.212724:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320502.212740:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320502.212763:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163983085312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320502.212831:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983085312:0@lo:38:kworker.0 00000100:00100000:3.0:1767320502.212901:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320502.213013:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085312 00000100:00100000:1.0:1767320502.213021:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085312 00000100:00100000:1.0:1767320502.213079:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2317 00000100:00100000:1.0:1767320502.213087:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983085312:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320502.213116:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320502.213167:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983085312:12345-0@lo:38:kworker.0 Request processed in 79us (265us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320502.213174:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2317 00000100:00080000:1.0:1767320502.213214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320502.213219:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320502.213223:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983085312:0@lo:38:kworker.0 00000100:00100000:0.0:1767320502.217309:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983085440:0@lo:400:kworker.0 00000100:00100000:0.0:1767320502.217385:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320502.217445:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983085184:0@lo:400:kworker.0 00000100:00100000:0.0:1767320502.217467:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320502.219371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320502.219381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320502.219386:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320502.219394:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5745 00000100:00080000:2.0:1767320502.219398:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320502.219406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320502.219955:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163983085568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320502.219978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320502.219981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320502.219983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320502.219986:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5745 00000100:00080000:2.0:1767320502.219989:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320502.219993:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320502.220008:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163983085696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320502.220041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320502.220045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320502.220047:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320502.220050:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5745 00000100:00080000:2.0:1767320502.220052:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320502.220056:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320502.220065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163983086080/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320502.220095:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983085952:0@lo:400:kworker.0 00000100:00100000:2.0:1767320502.220167:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320502.220201:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320502.220206:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983085824:0@lo:400:kworker.0 00000100:00100000:2.0:1767320502.220226:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320502.221843:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085952 00000100:00100000:3.0:1767320502.221856:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085952 00000100:00100000:3.0:1767320502.221941:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085824 00000100:00100000:3.0:1767320502.221945:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085824 00000100:00100000:3.0:1767320502.221973:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1081 00000100:00080000:2.0:1767320502.221978:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320502.221982:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320502.221982:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983085952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320502.222069:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983085952:12345-0@lo:400:kworker.0 Request processed in 88us (1900us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320502.222076:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1081 00000100:00100000:3.0:1767320502.222091:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082 00000100:00100000:3.0:1767320502.222097:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983085824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320502.222139:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983085824:12345-0@lo:400:kworker.0 Request processed in 42us (1912us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320502.222144:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082 00000100:00100000:3.0:1767320502.222168:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085440 00000100:00100000:3.0:1767320502.222172:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085440 00000100:00100000:3.0:1767320502.222206:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085184 00000100:00100000:3.0:1767320502.222209:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085184 00000100:00100000:3.0:1767320502.222227:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752 00000100:00100000:3.0:1767320502.222233:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983085440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320502.222306:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983085440:12345-0@lo:400:kworker.0 Request processed in 72us (4920us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320502.222312:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752 00000100:00100000:3.0:1767320502.222324:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1753 00000100:00100000:3.0:1767320502.222330:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983085184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320502.222356:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983085184:12345-0@lo:400:kworker.0 Request processed in 27us (4888us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320502.222362:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1753 00000100:00100000:1.0:1767320502.222623:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983085568:0@lo:38:kworker.0 00000100:00100000:1.0:1767320502.222664:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320502.222713:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983085696:0@lo:38:kworker.0 00000100:00100000:1.0:1767320502.222736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320502.222760:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983086080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320502.222780:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320502.222808:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085696 00000100:00100000:1.0:1767320502.222812:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085696 00000100:00100000:1.0:1767320502.222831:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 604 00000100:00100000:1.0:1767320502.222836:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983085696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320502.222852:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320502.222886:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983085696:12345-0@lo:38:kworker.0 Request processed in 49us (149us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320502.222892:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 604 00000100:00100000:1.0:1767320502.222909:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983085568 00000100:00100000:1.0:1767320502.222913:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983085568 00000100:00100000:1.0:1767320502.222929:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086080 00000100:00100000:1.0:1767320502.222932:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086080 00000100:00100000:1.0:1767320502.222947:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2318 00000100:00100000:1.0:1767320502.222950:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983085568:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320502.222959:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320502.222983:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983085568:12345-0@lo:38:kworker.0 Request processed in 31us (319us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320502.222988:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2318 00000100:00100000:1.0:1767320502.222998:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2319 00000100:00100000:1.0:1767320502.223001:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983086080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320502.223008:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320502.223027:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983086080:12345-0@lo:38:kworker.0 Request processed in 25us (247us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320502.223032:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2319 00000100:00080000:1.0:1767320502.223062:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320502.223067:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320502.223082:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983085568:0@lo:38:kworker.0 00000100:00080000:1.0:1767320502.223091:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320502.223093:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320502.223096:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983085696:0@lo:38:kworker.0 00000100:00080000:1.0:1767320502.223113:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320502.223117:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320502.223120:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983086080:0@lo:38:kworker.0 00010000:00080000:0.0:1767320502.224933:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594700 x1853163983085440/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320518 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320502.224951:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983085440:0@lo:400:kworker.0 00010000:00080000:0.0:1767320502.224966:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597100 x1853163983085184/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320518 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320502.224975:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983085184:0@lo:400:kworker.0 00000100:00100000:2.0:1767320502.225684:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983085824:0@lo:400:kworker.0 00000100:00100000:2.0:1767320502.225693:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983085952:0@lo:400:kworker.0 00000100:00080000:2.0:1767320502.225714:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320502.405191:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320503.428640:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320503.748584:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320503.748607:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983086208:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320503.748692:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320503.748940:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086208 00000100:00100000:1.0:1767320503.748949:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086208 00000100:00100000:1.0:1767320503.749008:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1174 00000100:00100000:1.0:1767320503.749017:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983086208:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320503.749048:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320503.749117:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983086208:12345-0@lo:13:osp-pre-0-1.0 Request processed in 101us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320503.749125:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1174 00000100:00100000:2.0:1767320503.749340:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320503.749352:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983086336:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320503.749384:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320503.749470:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086336 00000100:00100000:1.0:1767320503.749476:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086336 00000100:00100000:1.0:1767320503.749507:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175 00000100:00100000:1.0:1767320503.749513:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983086336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320503.749531:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320503.749609:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983086208:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320503.750610:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983086336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1090us (1218us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320503.750622:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175 00000100:00100000:2.0:1767320503.753845:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983086336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320504.206134:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320504.206184:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320504.206324:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685248 00000100:00100000:0.0:1767320504.206336:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685248 00000100:00100000:0.0:1767320504.206409:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685120 00000100:00100000:0.0:1767320504.206413:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685120 00000100:00100000:0.0:1767320504.206434:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1754 00000100:00100000:0.0:1767320504.206447:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915685248:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320504.206454:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2320 00000100:00100000:3.0:1767320504.206466:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915685120:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320504.206487:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320504.206532:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915685248:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320504.206541:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915685120:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 73us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320504.206546:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1754 00000100:00100000:3.0:1767320504.206555:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2320 00000100:00080000:2.0:1767320504.206638:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320504.453480:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320505.481440:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320506.500602:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320507.332741:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320507.332752:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320507.332819:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320507.332824:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320507.332827:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320507.332837:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5750 00000100:00080000:3.0:1767320507.332843:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320507.332850:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320507.332861:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983086464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320507.332874:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2d80 x1853163983086592/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320507.332910:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320507.332913:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320507.332916:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320507.332920:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5750 00000100:00080000:3.0:1767320507.332922:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320507.332927:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320507.332938:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320507.332940:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1c00 x1853163983086848/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320507.332951:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320507.332955:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320507.332956:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320507.332960:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5750 00000100:00080000:3.0:1767320507.332963:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320507.332966:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320507.332978:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1500 x1853163983086976/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320507.333024:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320507.333032:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983086720:0@lo:400:kworker.0 00000100:00080000:3.0:1767320507.333047:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320507.333051:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320507.333054:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320507.333058:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5750 00000100:00100000:0.0:1767320507.333062:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320507.333091:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320507.333096:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320507.333107:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983086592:0@lo:38:kworker.0 00000100:00100000:3.0:1767320507.333112:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3480 x1853163983087360/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320507.333115:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086464 00000100:00100000:0.0:1767320507.333124:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086464 00000100:00100000:3.0:1767320507.333144:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983087104:0@lo:400:kworker.0 00000100:00100000:1.0:1767320507.333153:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320507.333189:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086720 00000100:00100000:0.0:1767320507.333194:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086720 00000100:00100000:1.0:1767320507.333197:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983086848:0@lo:38:kworker.0 00000100:00100000:0.0:1767320507.333224:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086592 00000100:00100000:0.0:1767320507.333228:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086592 00000100:00100000:1.0:1767320507.333233:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320507.333244:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086848 00000100:00100000:0.0:1767320507.333248:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086848 00000100:00100000:1.0:1767320507.333249:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983086976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320507.333263:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1755 00000100:00100000:1.0:1767320507.333308:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320507.333313:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983086464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320507.333345:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983087360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320507.333374:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983086464:12345-0@lo:400:kworker.0 Request processed in 101us (436us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320507.333383:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320507.333388:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1755 00000100:00100000:0.0:1767320507.333407:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087360 00000100:00100000:0.0:1767320507.333410:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087360 00000100:00100000:0.0:1767320507.333424:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756 00000100:00080000:1.0:1767320507.333425:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320507.333433:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983086720:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320507.333445:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2321 00000100:00100000:1.0:1767320507.333453:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983086592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320507.333472:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320507.333478:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983086720:12345-0@lo:400:kworker.0 Request processed in 46us (415us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320507.333489:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756 00000100:00100000:0.0:1767320507.333516:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2322 00000100:00100000:1.0:1767320507.333518:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983086592:12345-0@lo:38:kworker.0 Request processed in 63us (367us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320507.333525:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983086848:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320507.333533:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2321 00010000:00020000:0.0:1767320507.333542:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320507.333554:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2323 00000100:00100000:2.0:1767320507.333569:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983087360:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320507.333575:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983086848:12345-0@lo:38:kworker.0 Request processed in 48us (341us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767320507.333584:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320507.333622:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983087360:12345-0@lo:38:kworker.0 Request processed in 60us (243us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320507.333666:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320507.333690:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320507.333696:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983087232:0@lo:400:kworker.0 00000100:00100000:3.0:1767320507.333719:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320507.333747:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983086976 00000100:00100000:3.0:1767320507.333752:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983086976 00000100:00100000:3.0:1767320507.333774:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 605 00000100:00100000:3.0:1767320507.333779:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983086976:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320507.333789:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320507.333815:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983086976:12345-0@lo:38:kworker.0 Request processed in 35us (508us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320507.333821:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 605 00000100:00100000:3.0:1767320507.333841:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087104 00000100:00100000:3.0:1767320507.333844:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087104 00000100:00100000:3.0:1767320507.333873:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087232 00000100:00100000:3.0:1767320507.333876:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087232 00000100:00100000:3.0:1767320507.333897:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1767320507.333903:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983087104:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320507.333935:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983087104:12345-0@lo:400:kworker.0 Request processed in 33us (749us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320507.333943:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1767320507.333960:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084 00000100:00100000:3.0:1767320507.333966:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983087232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320507.333994:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983087232:12345-0@lo:400:kworker.0 Request processed in 29us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320507.334000:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084 00000100:00080000:3.0:1767320507.334039:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320507.334042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320507.334066:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983087232:0@lo:400:kworker.0 00000100:00100000:3.0:1767320507.334077:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983087104:0@lo:400:kworker.0 00010000:00080000:3.0:1767320507.334134:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1180 x1853163983086720/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320523 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320507.334144:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983086720:0@lo:400:kworker.0 00000100:00080000:1.0:1767320507.334155:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00080000:3.0:1767320507.334157:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f3b80 x1853163983086464/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320523 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320507.334165:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320507.334171:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983086464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320507.334174:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983087360:0@lo:38:kworker.0 00000100:00080000:1.0:1767320507.334195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320507.334199:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320507.334203:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983086592:0@lo:38:kworker.0 00000100:00080000:1.0:1767320507.334214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320507.334217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320507.334221:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983086848:0@lo:38:kworker.0 00000100:00080000:1.0:1767320507.334230:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320507.334234:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320507.334237:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983086976:0@lo:38:kworker.0 00000100:00100000:0.0:1767320507.336200:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2322 00000100:00100000:2.0:1767320507.341687:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2323 00010000:00010000:2.0:1767320507.532547:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320508.548658:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320508.868826:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983087616:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320508.868898:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320508.868960:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320508.868967:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983087488:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320508.869000:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320508.869033:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087616 00000100:00100000:2.0:1767320508.869041:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087616 00000100:00100000:2.0:1767320508.869087:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087488 00000100:00100000:2.0:1767320508.869091:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087488 00000100:00100000:2.0:1767320508.869107:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176 00000100:00100000:2.0:1767320508.869114:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983087616:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320508.869143:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320508.869196:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983087616:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320508.869203:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176 00000100:00100000:2.0:1767320508.869234:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1177 00000100:00100000:2.0:1767320508.869241:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983087488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320508.869255:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320508.869305:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983087488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (304us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320508.869312:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1177 00000100:00100000:2.0:1767320508.869360:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983087488:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320508.869372:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983087616:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320509.325950:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320509.326053:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320509.326180:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685504 00000100:00100000:1.0:1767320509.326196:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685504 00000100:00100000:1.0:1767320509.326408:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685376 00000100:00100000:1.0:1767320509.326413:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685376 00000100:00080000:3.0:1767320509.326451:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320509.326853:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1757 00000100:00100000:2.0:1767320509.326869:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915685504:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320509.327048:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915685504:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 183us (1114us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320509.327056:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1757 00000100:00100000:2.0:1767320509.327092:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2324 00000100:00100000:2.0:1767320509.327097:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915685376:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320509.327118:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320509.327153:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915685376:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (1101us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320509.327159:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2324 00010000:00010000:2.0:1767320509.576707:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320510.596702:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320511.621323:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320512.458167:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320512.458180:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320512.458309:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320512.458315:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320512.458319:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320512.458331:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5755 00000100:00080000:2.0:1767320512.458338:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320512.458348:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320512.458350:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320512.458376:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983087744:0@lo:400:kworker.0 00000100:00100000:2.0:1767320512.458378:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597100 x1853163983087872/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320512.458417:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320512.458421:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320512.458423:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320512.458427:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5755 00000100:00080000:2.0:1767320512.458430:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320512.458434:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320512.458448:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163983088128/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320512.458457:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320512.458461:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320512.458463:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320512.458467:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5755 00000100:00080000:2.0:1767320512.458469:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320512.458476:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320512.458480:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320512.458497:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163983088256/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320512.458505:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983087872:0@lo:38:kworker.0 00000100:00080000:2.0:1767320512.458543:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320512.458548:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1767320512.458552:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320512.458553:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320512.458561:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5755 00000100:00080000:2.0:1767320512.458563:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320512.458569:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983088000:0@lo:400:kworker.0 00000100:00080000:2.0:1767320512.458569:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320512.458589:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983088128:0@lo:38:kworker.0 00000100:00100000:2.0:1767320512.458590:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff800 x1853163983088640/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320512.458601:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320512.458624:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087744 00000100:00100000:2.0:1767320512.458632:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983087872 00000100:00100000:1.0:1767320512.458634:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087744 00000100:00100000:2.0:1767320512.458638:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983087872 00000100:00100000:0.0:1767320512.458642:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320512.458664:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983088256:0@lo:38:kworker.0 00000100:00100000:1.0:1767320512.458687:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088128 00000100:00100000:2.0:1767320512.458687:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088000 00000100:00100000:2.0:1767320512.458691:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088000 00000100:00100000:1.0:1767320512.458693:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088128 00000100:00100000:0.0:1767320512.458694:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320512.458712:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1758 00000100:00100000:0.0:1767320512.458720:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983088640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320512.458724:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983087744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320512.458731:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1759 00000100:00100000:2.0:1767320512.458741:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983088000:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320512.458747:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320512.458773:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088640 00000100:00100000:0.0:1767320512.458777:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088640 00000100:00100000:1.0:1767320512.458788:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983087744:12345-0@lo:400:kworker.0 Request processed in 67us (313us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320512.458798:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1758 00000100:00100000:2.0:1767320512.458799:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983088000:12345-0@lo:400:kworker.0 Request processed in 57us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320512.458862:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983088384:0@lo:400:kworker.0 00000100:00100000:3.0:1767320512.458899:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320512.458940:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320512.458949:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983088512:0@lo:400:kworker.0 00000100:00100000:3.0:1767320512.458976:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320512.459001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320512.459027:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088256 00000100:00100000:3.0:1767320512.459032:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088256 00000100:00100000:3.0:1767320512.459054:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 606 00000100:00100000:3.0:1767320512.459061:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983088256:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320512.459091:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320512.459135:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983088256:12345-0@lo:38:kworker.0 Request processed in 68us (436us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320512.459142:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 606 00000100:00100000:3.0:1767320512.459164:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088384 00000100:00100000:3.0:1767320512.459167:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088384 00000100:00100000:3.0:1767320512.459195:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088512 00000100:00100000:3.0:1767320512.459198:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088512 00000100:00100000:3.0:1767320512.459217:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085 00000100:00100000:3.0:1767320512.459229:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983088384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320512.459263:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983088384:12345-0@lo:400:kworker.0 Request processed in 40us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320512.459287:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085 00000100:00100000:3.0:1767320512.459300:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086 00000100:00100000:3.0:1767320512.459306:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983088512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320512.459332:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983088512:12345-0@lo:400:kworker.0 Request processed in 26us (356us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320512.459338:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086 00000100:00080000:3.0:1767320512.459366:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320512.459369:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320512.459399:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983088512:0@lo:400:kworker.0 00000100:00100000:3.0:1767320512.459410:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983088384:0@lo:400:kworker.0 00000100:00100000:0.0:1767320512.459799:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2325 00000100:00100000:0.0:1767320512.459807:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983087872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320512.459822:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320512.459857:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983087872:12345-0@lo:38:kworker.0 Request processed in 49us (1305us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320512.459863:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2325 00000100:00100000:0.0:1767320512.459878:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2326 00000100:00100000:0.0:1767320512.459883:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983088128:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320512.459893:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320512.459984:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2327 00000100:00100000:1.0:1767320512.459992:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983088640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320512.460011:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320512.460040:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983088640:12345-0@lo:38:kworker.0 Request processed in 48us (1296us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320512.460046:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2327 00000100:00080000:1.0:1767320512.460099:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320512.460107:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320512.460110:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983088256:0@lo:38:kworker.0 00000100:00080000:1.0:1767320512.460119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320512.460123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320512.460126:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983088640:0@lo:38:kworker.0 00000100:00080000:1.0:1767320512.460135:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320512.460139:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320512.460142:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983087872:0@lo:38:kworker.0 00000100:00080000:1.0:1767320512.460151:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320512.460154:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320512.460157:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983088128:0@lo:38:kworker.0 00010000:00080000:1.0:1767320512.460210:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594700 x1853163983088000/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320528 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320512.460221:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983088000:0@lo:400:kworker.0 00010000:00080000:1.0:1767320512.460233:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596300 x1853163983087744/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320528 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320512.460241:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983087744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320512.460975:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983088128:12345-0@lo:38:kworker.0 Request processed in 31us (1274us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320512.460986:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2326 00000100:00100000:2.0:1767320512.465286:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1759 00010000:00010000:2.0:1767320512.646338:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320513.668595:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320513.990693:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983088768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320513.990811:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320513.991326:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088768 00000100:00100000:0.0:1767320513.991338:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088768 00000100:00100000:0.0:1767320513.991415:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1178 00000100:00100000:0.0:1767320513.991427:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983088768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320513.991471:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320513.991534:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983088768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 109us (722us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320513.991543:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1178 00000100:00100000:3.0:1767320513.992159:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983088768:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320513.994216:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983088896:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320513.994343:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320514.000578:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983088896 00000100:00100000:3.0:1767320514.000595:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983088896 00000100:00100000:3.0:1767320514.000757:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179 00000100:00100000:3.0:1767320514.000768:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983088896:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320514.000802:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320514.001802:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320514.002114:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983088896:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1341us (7764us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320514.002124:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179 00000100:00100000:2.0:1767320514.003104:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983088896:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320514.448030:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320514.448073:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320514.448101:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915685888 offset 0 length 520: 4 00000100:00100000:3.0:1767320514.448187:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685632 00000100:00100000:3.0:1767320514.448198:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685632 00000100:00100000:3.0:1767320514.448248:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915685760 00000100:00100000:3.0:1767320514.448252:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915685760 00000100:00100000:3.0:1767320514.448347:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1760 00000100:00100000:3.0:1767320514.448357:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915685760:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320514.448439:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915685760:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (364us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320514.448451:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1760 00000100:00100000:3.0:1767320514.448486:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2328 00000100:00100000:3.0:1767320514.448491:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915685632:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320514.448518:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320514.448549:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915685632:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (530us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320514.448556:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2328 00000100:00080000:1.0:1767320514.448655:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320514.692606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320515.717035:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320516.548537:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320491/real 1767320491] req@ffff934638597800 x1853163983082496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320516 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320516.548584:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934638597800 x1853163983082496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320516 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320516.548611:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320516.548650:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320516.548655:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983082496:0@lo:250:kworker.0 00000100:00080000:2.0:1767320516.549150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.549158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320516.549162:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.549168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5740 00000100:00080000:2.0:1767320516.549175:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.549184:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320516.549241:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163983089024/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.549352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.549357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.549359:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.549364:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5760 00000100:00080000:2.0:1767320516.549366:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.549371:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320516.549387:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff480 x1853163983089280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.549405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.549408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.549410:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.549414:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5760 00000100:00080000:2.0:1767320516.549417:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.549421:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320516.549431:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fca80 x1853163983089536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.549441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.549444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.549446:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.549450:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5760 00000100:00080000:2.0:1767320516.549453:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.549457:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320516.549468:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fdf80 x1853163983089664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320516.549501:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320516.549504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320516.549506:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320516.549509:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5760 00000100:00080000:2.0:1767320516.549512:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320516.549515:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320516.549524:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ffb80 x1853163983090048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320516.549559:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983089920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320516.549643:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320516.549723:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320516.549728:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983089792:0@lo:400:kworker.0 00000100:00100000:2.0:1767320516.549748:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320516.549911:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983089408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.550067:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983089024:0@lo:250:kworker.0 00000400:00000100:0.0:1767320516.550112:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983089024 offset 0 length 520: 4 00000100:00100000:3.0:1767320516.550182:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983089152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.550713:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983089280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320516.550754:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320516.550783:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983089536:0@lo:38:kworker.0 00000100:00100000:0.0:1767320516.550808:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320516.550836:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983089664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320516.550852:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320516.550993:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089280 00000100:00100000:2.0:1767320516.551003:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089280 00000100:00100000:2.0:1767320516.551041:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089536 00000100:00100000:2.0:1767320516.551045:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089536 00000100:00100000:2.0:1767320516.551060:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089408 00000100:00100000:2.0:1767320516.551063:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089408 00000100:00100000:2.0:1767320516.551100:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1761 00000100:00100000:2.0:1767320516.551109:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983089408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.553676:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320516.553769:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089920 00000100:00100000:3.0:1767320516.553779:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089920 00000100:00100000:3.0:1767320516.553852:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089792 00000100:00100000:3.0:1767320516.553856:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089792 00000100:00100000:3.0:1767320516.553879:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1087 00000100:00100000:3.0:1767320516.553889:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983089920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.553971:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983089920:12345-0@lo:400:kworker.0 Request processed in 83us (4327us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320516.553981:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1087 00000100:00100000:3.0:1767320516.554010:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088 00000100:00100000:3.0:1767320516.554017:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983089792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.554055:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983089792:12345-0@lo:400:kworker.0 Request processed in 39us (4305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320516.554061:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088 00000100:00100000:3.0:1767320516.554087:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089152 00000100:00100000:3.0:1767320516.554091:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089152 00000100:00100000:3.0:1767320516.554124:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1762 00000100:00100000:3.0:1767320516.554130:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983089152:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.554167:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983089152:12345-0@lo:400:kworker.0 Request processed in 37us (501us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320516.554173:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1762 00000100:00100000:3.0:1767320516.554185:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2329 00000100:00100000:3.0:1767320516.554474:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983089280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320516.554498:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320516.554533:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983089280:12345-0@lo:38:kworker.0 Request processed in 59us (3778us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320516.554539:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2329 00000100:00100000:3.0:1767320516.554555:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2330 00000100:00100000:3.0:1767320516.554559:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983089536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320516.554568:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320516.554590:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983089536:12345-0@lo:38:kworker.0 Request processed in 30us (3782us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320516.554595:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2330 00000100:00100000:3.0:1767320516.554654:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983089792:0@lo:400:kworker.0 00000100:00100000:3.0:1767320516.554669:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983089920:0@lo:400:kworker.0 00010000:00080000:3.0:1767320516.554707:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc380 x1853163983089152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320532 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320516.554717:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983089152:0@lo:400:kworker.0 00000100:00080000:3.0:1767320516.554914:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320516.554919:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320516.554922:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320516.556253:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983089408:12345-0@lo:400:kworker.0 Request processed in 5072us (5345us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320516.556286:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1761 00010000:00080000:1.0:1767320516.556487:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd500 x1853163983089408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320532 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320516.556501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983089408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320516.558692:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320516.558941:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983090048:0@lo:38:kworker.0 00000100:00100000:0.0:1767320516.559079:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320516.559454:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320516.559461:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320516.559465:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983089280:0@lo:38:kworker.0 00000100:00080000:0.0:1767320516.559480:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320516.559483:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320516.559487:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983089536:0@lo:38:kworker.0 00000100:00100000:3.0:1767320516.559567:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983089664 00000100:00100000:3.0:1767320516.559576:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983089664 00000100:00100000:2.0:1767320516.559604:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090048 00000100:00100000:2.0:1767320516.559620:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090048 00000100:00100000:3.0:1767320516.559641:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 607 00000100:00100000:3.0:1767320516.559652:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983089664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320516.559678:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320516.559691:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2331 00000100:00100000:2.0:1767320516.559701:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090048:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320516.559726:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320516.559740:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983089664:12345-0@lo:38:kworker.0 Request processed in 87us (1058us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320516.559749:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 607 00000100:00100000:2.0:1767320516.559817:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090048:12345-0@lo:38:kworker.0 Request processed in 114us (735us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320516.559831:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2331 00000100:00080000:0.0:1767320516.560596:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320516.560604:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320516.560608:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983089664:0@lo:38:kworker.0 00000100:00080000:0.0:1767320516.560619:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320516.560622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320516.560625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983090048:0@lo:38:kworker.0 00010000:00010000:2.0:1767320516.740808:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320517.764530:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320518.788526:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320519.109814:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983090176:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320519.109886:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320519.109971:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090176 00000100:00100000:1.0:1767320519.109979:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090176 00000100:00100000:1.0:1767320519.110042:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1180 00000100:00100000:1.0:1767320519.110050:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983090176:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320519.110078:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320519.110119:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983090176:12345-0@lo:13:osp-pre-0-1.0 Request processed in 70us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320519.110126:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1180 00000100:00080000:3.0:1767320519.110530:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320519.110569:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983090176:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320519.114419:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320519.114443:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983090304:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320519.114512:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320519.114606:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090304 00000100:00100000:0.0:1767320519.114614:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090304 00000100:00100000:0.0:1767320519.114681:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181 00000100:00100000:0.0:1767320519.114691:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983090304:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320519.114720:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320519.114763:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983090304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (251us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320519.114770:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181 00000100:00080000:3.0:1767320519.114927:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320519.114965:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983090304:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320519.565104:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320519.565213:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320519.577750:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686016 00000100:00100000:0.0:1767320519.577765:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686016 00000100:00100000:0.0:1767320519.577910:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686144 00000100:00100000:0.0:1767320519.577914:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686144 00000100:00100000:1.0:1767320519.578033:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1763 00000100:00100000:1.0:1767320519.578045:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915686144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320519.578129:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915686144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (12916us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320519.578137:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1763 00000100:00100000:1.0:1767320519.578155:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2332 00000100:00100000:1.0:1767320519.578159:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915686016:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320519.578179:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320519.578205:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915686016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (13112us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320519.578211:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2332 00000100:00080000:2.0:1767320519.578285:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320519.816193:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320520.836518:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320521.668669:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.668678:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320521.668732:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.668737:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.668740:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.668756:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.668761:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.668767:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320521.668789:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163983090560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.668814:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.668817:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.668819:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.668822:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.668824:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.668828:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320521.668836:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2300 x1853163983090816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.668845:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.668848:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.668850:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.668853:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.668855:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.668858:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320521.668865:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0a80 x1853163983090944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320521.668889:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320521.668893:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320521.668895:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320521.668898:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320521.668901:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320521.668904:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320521.668914:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163983091328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320521.668981:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983091200:0@lo:400:kworker.0 00000100:00100000:0.0:1767320521.669039:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983090560:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.669050:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320521.669069:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320521.669092:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983090816:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.669094:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320521.669099:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983091072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320521.669112:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.669114:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320521.669125:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983090944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.669135:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983090688:0@lo:400:kworker.0 00000100:00100000:0.0:1767320521.669145:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.669153:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320521.669162:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983091328:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.669168:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320521.669173:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983090432:0@lo:400:kworker.0 00000100:00100000:0.0:1767320521.669181:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320521.669202:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320521.669218:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090944 00000100:00100000:0.0:1767320521.669225:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090944 00000100:00100000:2.0:1767320521.669233:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090560 00000100:00100000:2.0:1767320521.669235:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090560 00000100:00100000:2.0:1767320521.669248:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090816 00000100:00100000:2.0:1767320521.669249:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090816 00000100:00100000:0.0:1767320521.669251:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 608 00000100:00100000:2.0:1767320521.669256:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090688 00000100:00100000:2.0:1767320521.669257:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090688 00000100:00100000:0.0:1767320521.669258:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983090944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320521.669293:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.669299:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091328 00000100:00100000:2.0:1767320521.669301:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091328 00000100:00100000:2.0:1767320521.669323:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983090432 00000100:00100000:2.0:1767320521.669326:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983090432 00000100:00100000:0.0:1767320521.669333:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983090944:12345-0@lo:38:kworker.0 Request processed in 74us (186us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320521.669340:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 608 00000100:00100000:2.0:1767320521.669343:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764 00000100:00100000:2.0:1767320521.669348:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983090688:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320521.669371:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320521.669376:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320521.669379:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983090944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320521.669398:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983090688:12345-0@lo:400:kworker.0 Request processed in 50us (243us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320521.669404:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764 00000100:00080000:0.0:1767320521.669409:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320521.669416:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1765 00000100:00100000:2.0:1767320521.669422:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983090432:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320521.669448:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983090432:12345-0@lo:400:kworker.0 Request processed in 26us (245us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320521.669455:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1765 00000100:00100000:2.0:1767320521.669466:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2333 00000100:00100000:2.0:1767320521.669470:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090560:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.669479:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.669512:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090560:12345-0@lo:38:kworker.0 Request processed in 40us (442us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.669518:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2333 00000100:00100000:2.0:1767320521.669530:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2334 00000100:00100000:2.0:1767320521.669533:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090816:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.669542:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.669565:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983090816:12345-0@lo:38:kworker.0 Request processed in 30us (452us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.669569:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2334 00000100:00100000:2.0:1767320521.669579:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2335 00000100:00100000:2.0:1767320521.669582:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983091328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320521.669589:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320521.669608:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983091328:12345-0@lo:38:kworker.0 Request processed in 24us (426us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320521.669613:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2335 00010000:00080000:2.0:1767320521.669636:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3100 x1853163983090432/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320537 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320521.669647:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983090432:0@lo:400:kworker.0 00010000:00080000:2.0:1767320521.669657:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3480 x1853163983090688/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320537 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320521.669665:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983090688:0@lo:400:kworker.0 00000100:00080000:0.0:1767320521.669675:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320521.669679:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320521.669682:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983090560:0@lo:38:kworker.0 00000100:00080000:0.0:1767320521.669691:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320521.669694:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320521.669697:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983090816:0@lo:38:kworker.0 00000100:00080000:0.0:1767320521.669704:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320521.669707:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320521.669709:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983091328:0@lo:38:kworker.0 00000100:00100000:3.0:1767320521.669872:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091200 00000100:00100000:3.0:1767320521.669877:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091200 00000100:00100000:3.0:1767320521.669920:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091072 00000100:00100000:3.0:1767320521.669923:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091072 00000100:00100000:3.0:1767320521.669945:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089 00000100:00100000:3.0:1767320521.669952:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983091200:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320521.669990:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983091200:12345-0@lo:400:kworker.0 Request processed in 39us (939us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320521.669996:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089 00000100:00080000:0.0:1767320521.670001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320521.670004:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320521.670008:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1090 00000100:00100000:3.0:1767320521.670014:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983091072:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320521.670040:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983091072:12345-0@lo:400:kworker.0 Request processed in 27us (924us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320521.670045:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1090 00000100:00100000:0.0:1767320521.670056:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983091072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320521.670066:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983091200:0@lo:400:kworker.0 00010000:00010000:2.0:1767320521.860596:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320522.884755:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320523.910090:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320524.228573:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320524.228596:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983091456:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320524.228670:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320524.228836:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091456 00000100:00100000:0.0:1767320524.228845:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091456 00000100:00100000:0.0:1767320524.228925:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1182 00000100:00100000:0.0:1767320524.228936:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983091456:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320524.228987:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320524.229034:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983091456:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320524.229041:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1182 00000100:00080000:2.0:1767320524.229413:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320524.229450:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983091456:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320524.229594:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320524.229605:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983091584:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320524.229639:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320524.229945:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091584 00000100:00100000:0.0:1767320524.229962:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091584 00000100:00100000:0.0:1767320524.230209:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1183 00000100:00100000:0.0:1767320524.230217:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983091584:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320524.230241:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320524.230380:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983091584:12345-0@lo:13:osp-pre-0-1.0 Request processed in 164us (740us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320524.230388:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1183 00000100:00080000:2.0:1767320524.230476:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320524.232726:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983091584:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320524.686216:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320524.686313:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320524.686601:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686400 00000100:00100000:0.0:1767320524.686614:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686400 00000100:00100000:0.0:1767320524.686702:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686272 00000100:00100000:0.0:1767320524.686706:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686272 00000100:00100000:0.0:1767320524.686732:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1766 00000100:00100000:3.0:1767320524.686745:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2336 00000100:00100000:0.0:1767320524.686746:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915686400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320524.686758:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915686272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320524.686782:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320524.686862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320524.686943:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915686400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 201us (738us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320524.686951:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1766 00000100:00100000:3.0:1767320524.688292:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915686272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 1529us (2016us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320524.688303:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2336 00010000:00010000:2.0:1767320524.932906:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320525.956677:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320526.789569:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320526.789581:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320526.789645:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320526.789649:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320526.789661:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320526.789671:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5770 00000100:00080000:2.0:1767320526.789676:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320526.789684:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320526.789687:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320526.789705:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983091712:0@lo:400:kworker.0 00000100:00100000:2.0:1767320526.789710:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163983091840/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320526.789760:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320526.789764:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320526.789766:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320526.789771:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5770 00000100:00080000:2.0:1767320526.789774:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320526.789777:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320526.789786:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320526.789793:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163983092096/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320526.789802:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320526.789806:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320526.789808:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320526.789811:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5770 00000100:00080000:2.0:1767320526.789813:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320526.789818:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320526.789833:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583b80 x1853163983092224/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320526.789836:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983091968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320526.789872:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320526.789885:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320526.789889:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320526.789891:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320526.789896:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5770 00000100:00080000:2.0:1767320526.789899:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320526.789902:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320526.789915:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163983092608/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320526.789921:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983092480:0@lo:400:kworker.0 00000100:00100000:0.0:1767320526.789968:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320526.790002:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320526.790007:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983092352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320526.790168:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091712 00000100:00100000:3.0:1767320526.790177:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091712 00000100:00100000:3.0:1767320526.790234:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091968 00000100:00100000:3.0:1767320526.790238:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091968 00000100:00100000:2.0:1767320526.790261:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092480 00000100:00100000:2.0:1767320526.790267:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092480 00000100:00100000:3.0:1767320526.790301:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1767 00000100:00100000:3.0:1767320526.790313:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983091712:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320526.790333:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091 00000100:00100000:2.0:1767320526.790343:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983092480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320526.790378:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983091712:12345-0@lo:400:kworker.0 Request processed in 67us (591us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320526.790388:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1767 00000100:00100000:2.0:1767320526.790389:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983092480:12345-0@lo:400:kworker.0 Request processed in 46us (418us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320526.790399:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091 00000100:00100000:3.0:1767320526.790412:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768 00000100:00100000:3.0:1767320526.790421:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983091968:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320526.790423:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320526.790429:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320526.790455:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983091968:12345-0@lo:400:kworker.0 Request processed in 35us (583us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320526.790462:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768 00010000:00080000:2.0:1767320526.790487:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100ff800 x1853163983091968/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320542 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320526.790501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983091968:0@lo:400:kworker.0 00010000:00080000:2.0:1767320526.790512:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fea00 x1853163983091712/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320542 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320526.790520:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983091712:0@lo:400:kworker.0 00000100:00100000:1.0:1767320526.790595:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983091840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.790626:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320526.790662:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983092096:0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.790733:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320526.790757:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983092224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.790781:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320526.790826:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983092608:0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.790847:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320526.790876:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092224 00000100:00100000:1.0:1767320526.790880:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092224 00000100:00100000:1.0:1767320526.790904:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 609 00000100:00100000:1.0:1767320526.790910:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983092224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320526.790931:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320526.790965:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983092224:12345-0@lo:38:kworker.0 Request processed in 55us (183us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320526.790972:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 609 00000100:00100000:1.0:1767320526.790990:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983091840 00000100:00100000:1.0:1767320526.790994:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983091840 00000100:00100000:1.0:1767320526.791010:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092096 00000100:00100000:1.0:1767320526.791013:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092096 00000100:00100000:1.0:1767320526.791177:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092608 00000100:00100000:1.0:1767320526.791182:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092608 00000100:00100000:3.0:1767320526.791183:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2337 00000100:00100000:3.0:1767320526.791191:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983091840:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.791206:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2338 00010000:00020000:3.0:1767320526.791206:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320526.791215:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983092096:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320526.791228:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320526.791244:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983091840:12345-0@lo:38:kworker.0 Request processed in 50us (616us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320526.791251:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2337 00000100:00100000:1.0:1767320526.791290:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983092096:12345-0@lo:38:kworker.0 Request processed in 52us (532us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320526.791290:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2339 00000100:00100000:3.0:1767320526.791299:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983092608:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320526.791303:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2338 00010000:00020000:3.0:1767320526.791314:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320526.791345:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320526.791346:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983092608:12345-0@lo:38:kworker.0 Request processed in 47us (497us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320526.791354:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320526.791359:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2339 00000100:00100000:1.0:1767320526.791362:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983091840:0@lo:38:kworker.0 00000100:00080000:1.0:1767320526.791377:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320526.791381:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320526.791384:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983092096:0@lo:38:kworker.0 00000100:00080000:1.0:1767320526.791395:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320526.791399:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320526.791403:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983092224:0@lo:38:kworker.0 00000100:00080000:1.0:1767320526.791412:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320526.791415:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320526.791417:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983092608:0@lo:38:kworker.0 00000100:00100000:0.0:1767320526.791489:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320526.791526:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983092480:0@lo:400:kworker.0 00000100:00100000:1.0:1767320526.791624:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092352 00000100:00100000:1.0:1767320526.791628:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092352 00000100:00100000:1.0:1767320526.791685:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092 00000100:00100000:1.0:1767320526.791693:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983092352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320526.791737:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983092352:12345-0@lo:400:kworker.0 Request processed in 45us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320526.791743:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092 00000100:00100000:0.0:1767320526.791915:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983092352:0@lo:400:kworker.0 00000100:00080000:2.0:1767320526.791967:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320526.982687:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320528.004546:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320529.028580:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320529.348655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983092736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320529.348657:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983092864:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320529.350078:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320529.350342:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320529.365044:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092736 00000100:00100000:1.0:1767320529.365061:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092736 00000100:00100000:1.0:1767320529.365152:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092864 00000100:00100000:1.0:1767320529.365156:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092864 00000100:00100000:1.0:1767320529.365177:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184 00000100:00100000:1.0:1767320529.365186:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983092736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320529.365222:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320529.365326:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983092736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 141us (15252us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320529.365333:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184 00000100:00100000:1.0:1767320529.365349:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1185 00000100:00100000:1.0:1767320529.365354:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983092864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320529.365369:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320529.365398:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983092864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 44us (15058us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320529.365404:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1185 00000100:00100000:3.0:1767320529.365579:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983092864:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320529.366993:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320529.367003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320529.367049:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983092736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320529.804972:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320529.805090:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686656 00000100:00100000:1.0:1767320529.805102:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686656 00000100:00100000:1.0:1767320529.805180:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1769 00000100:00100000:1.0:1767320529.805190:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915686656:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320529.805268:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915686656:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 78us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320529.805309:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1769 00000100:00100000:3.0:1767320529.805355:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320529.805516:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686528 00000100:00100000:1.0:1767320529.805521:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686528 00000100:00100000:1.0:1767320529.805548:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2340 00000100:00100000:1.0:1767320529.805553:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915686528:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320529.805573:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320529.805614:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915686528:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (261us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320529.805621:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2340 00000100:00080000:2.0:1767320529.805734:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320530.054812:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320531.076705:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320531.908674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320531.910786:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320531.910853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320531.910858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320531.910862:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320531.910869:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5775 00000100:00080000:2.0:1767320531.910874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320531.910880:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320531.910898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580000 x1853163983093120/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320531.910928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320531.910932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320531.910934:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320531.910937:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5775 00000100:00080000:2.0:1767320531.910940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320531.910944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320531.910953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581c00 x1853163983093376/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320531.910957:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983092992:0@lo:400:kworker.0 00000100:00080000:2.0:1767320531.910966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320531.910971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320531.910973:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320531.910976:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5775 00000100:00080000:2.0:1767320531.910978:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320531.910981:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320531.910993:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580a80 x1853163983093504/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320531.911027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320531.911030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320531.911032:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320531.911036:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320531.911039:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5775 00000100:00080000:2.0:1767320531.911043:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320531.911046:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320531.911056:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582a00 x1853163983093888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320531.911083:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983093760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320531.911098:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320531.911107:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983093248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320531.911118:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320531.911135:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320531.911145:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320531.911153:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983093632:0@lo:400:kworker.0 00000100:00100000:2.0:1767320531.911178:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320531.911189:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983092992 00000100:00100000:3.0:1767320531.911196:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983092992 00000100:00100000:2.0:1767320531.911212:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093760 00000100:00100000:2.0:1767320531.911217:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093760 00000100:00100000:3.0:1767320531.911246:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093248 00000100:00100000:3.0:1767320531.911250:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093248 00000100:00100000:2.0:1767320531.911254:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093632 00000100:00100000:2.0:1767320531.911258:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093632 00000100:00100000:2.0:1767320531.911306:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1093 00000100:00100000:3.0:1767320531.911307:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1770 00000100:00100000:2.0:1767320531.911317:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983093760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320531.911320:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983092992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320531.911378:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983093760:12345-0@lo:400:kworker.0 Request processed in 63us (259us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320531.911384:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983092992:12345-0@lo:400:kworker.0 Request processed in 67us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320531.911389:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1093 00000100:00100000:3.0:1767320531.911397:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1770 00000100:00100000:2.0:1767320531.911412:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094 00000100:00100000:3.0:1767320531.911416:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1771 00000100:00100000:2.0:1767320531.911422:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983093632:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320531.911428:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983093248:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320531.911467:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983093632:12345-0@lo:400:kworker.0 Request processed in 47us (287us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320531.911470:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983093248:12345-0@lo:400:kworker.0 Request processed in 45us (334us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320531.911480:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094 00000100:00100000:3.0:1767320531.911482:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1771 00000100:00080000:2.0:1767320531.911505:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320531.911511:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320531.911514:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320531.911530:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583800 x1853163983093248/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320547 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320531.911541:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983093248:0@lo:400:kworker.0 00000100:00100000:2.0:1767320531.911543:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983093632:0@lo:400:kworker.0 00010000:00080000:3.0:1767320531.911556:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604581f80 x1853163983092992/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320547 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320531.911574:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983093760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320531.911578:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983092992:0@lo:400:kworker.0 00000100:00100000:1.0:1767320531.911642:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983093120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320531.911672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320531.911710:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983093376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320531.911729:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320531.911752:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983093504:0@lo:38:kworker.0 00000100:00100000:3.0:1767320531.911771:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093120 00000100:00100000:1.0:1767320531.911774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320531.911775:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093120 00000100:00100000:3.0:1767320531.911790:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093376 00000100:00100000:3.0:1767320531.911793:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093376 00000100:00100000:1.0:1767320531.911797:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983093888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320531.911804:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2341 00000100:00100000:3.0:1767320531.911808:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093120:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320531.911814:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767320531.911826:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320531.911841:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093504 00000100:00100000:1.0:1767320531.911845:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093504 00000100:00100000:3.0:1767320531.911858:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093120:12345-0@lo:38:kworker.0 Request processed in 49us (186us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320531.911864:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2341 00000100:00100000:1.0:1767320531.911865:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 610 00000100:00100000:1.0:1767320531.911869:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983093504:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320531.911873:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983093888 00000100:00100000:3.0:1767320531.911875:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983093888 00010000:00020000:1.0:1767320531.911879:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320531.911884:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2342 00000100:00100000:3.0:1767320531.911888:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093376:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320531.911895:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320531.911901:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983093504:12345-0@lo:38:kworker.0 Request processed in 32us (126us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320531.911907:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 610 00000100:00100000:3.0:1767320531.911911:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093376:12345-0@lo:38:kworker.0 Request processed in 23us (182us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320531.911916:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2342 00000100:00100000:3.0:1767320531.911925:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2343 00000100:00100000:3.0:1767320531.911928:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320531.911936:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767320531.911946:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320531.911951:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320531.911954:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983093120:0@lo:38:kworker.0 00000100:00100000:3.0:1767320531.911954:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983093888:12345-0@lo:38:kworker.0 Request processed in 25us (139us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320531.911960:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2343 00000100:00080000:1.0:1767320531.911962:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320531.911964:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320531.911967:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983093376:0@lo:38:kworker.0 00000100:00080000:1.0:1767320531.911975:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320531.911978:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320531.911980:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983093504:0@lo:38:kworker.0 00000100:00080000:1.0:1767320531.911987:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320531.911990:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320531.911992:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983093888:0@lo:38:kworker.0 00010000:00010000:2.0:1767320532.101448:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320533.124903:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320534.148850:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320534.470395:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983094016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320534.470474:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320534.470705:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094016 00000100:00100000:3.0:1767320534.470713:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094016 00000100:00100000:3.0:1767320534.470795:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1186 00000100:00100000:3.0:1767320534.470806:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983094016:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320534.470848:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320534.470898:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983094016:12345-0@lo:13:osp-pre-0-1.0 Request processed in 94us (423us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320534.470907:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1186 00000100:00080000:0.0:1767320534.473376:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320534.473459:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983094016:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320534.476076:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320534.476098:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983094144:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320534.476177:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320534.476378:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094144 00000100:00100000:1.0:1767320534.476394:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094144 00000100:00100000:1.0:1767320534.476474:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187 00000100:00100000:1.0:1767320534.476485:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983094144:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320534.476725:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320534.476964:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320534.477030:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983094144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 548us (852us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320534.477038:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187 00000100:00100000:0.0:1767320534.477128:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983094144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320534.923830:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320534.923885:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320534.924069:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686912 00000100:00100000:0.0:1767320534.924079:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686912 00000100:00100000:0.0:1767320534.924169:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915686784 00000100:00100000:3.0:1767320534.924267:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1772 00000100:00100000:3.0:1767320534.924309:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915686912:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320534.924383:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915686912:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320534.924394:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1772 00000100:00100000:0.0:1767320534.924620:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915686784 00000100:00100000:0.0:1767320534.924650:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2344 00000100:00100000:0.0:1767320534.924655:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915686784:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320534.924675:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320534.924841:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915686784:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 184us (954us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320534.924848:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2344 00000100:00080000:1.0:1767320534.927007:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320535.175798:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320536.197257:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320537.028543:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320537.028554:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320537.028616:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320537.028620:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320537.028624:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320537.028632:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5780 00000100:00080000:3.0:1767320537.028636:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320537.028642:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320537.028666:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163983094400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320537.028696:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320537.028700:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320537.028701:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320537.028704:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5780 00000100:00080000:3.0:1767320537.028707:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320537.028710:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320537.028720:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014a80 x1853163983094656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320537.028728:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320537.028731:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320537.028733:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320537.028736:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5780 00000100:00080000:3.0:1767320537.028738:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320537.028742:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320537.028754:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163983094784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320537.028784:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320537.028788:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320537.028790:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320537.028793:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5780 00000100:00080000:3.0:1767320537.028795:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320537.028799:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320537.028809:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015f80 x1853163983095168/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320537.028836:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983095040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.028915:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.028973:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320537.028979:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983094912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.029001:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320537.029031:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095040 00000100:00100000:3.0:1767320537.029039:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095040 00000100:00100000:3.0:1767320537.029088:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094912 00000100:00100000:3.0:1767320537.029091:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094912 00000100:00100000:3.0:1767320537.029116:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767320537.029124:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983095040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.029170:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983095040:12345-0@lo:400:kworker.0 Request processed in 46us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320537.029178:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1767320537.029193:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096 00000100:00100000:3.0:1767320537.029199:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983094912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.029224:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983094912:12345-0@lo:400:kworker.0 Request processed in 26us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320537.029230:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096 00000100:00080000:3.0:1767320537.029247:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320537.029251:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320537.029320:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983094912:0@lo:400:kworker.0 00000100:00100000:3.0:1767320537.029332:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983095040:0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.029427:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983094272:0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.029458:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320537.029498:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320537.029504:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983094528:0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.029524:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320537.029550:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094272 00000100:00100000:0.0:1767320537.029554:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094272 00000100:00100000:0.0:1767320537.029583:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094528 00000100:00100000:0.0:1767320537.029586:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094528 00000100:00100000:0.0:1767320537.029605:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1773 00000100:00100000:0.0:1767320537.029612:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983094272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.029645:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983094272:12345-0@lo:400:kworker.0 Request processed in 34us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320537.029651:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1773 00000100:00100000:0.0:1767320537.029663:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1774 00000100:00100000:0.0:1767320537.029669:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983094528:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320537.029697:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983094528:12345-0@lo:400:kworker.0 Request processed in 28us (171us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320537.029702:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1774 00010000:00080000:0.0:1767320537.029729:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016680 x1853163983094528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320553 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320537.029739:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983094528:0@lo:400:kworker.0 00010000:00080000:0.0:1767320537.029748:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596a00 x1853163983094272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320553 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320537.029756:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983094272:0@lo:400:kworker.0 00000100:00100000:1.0:1767320537.030559:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983094400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320537.030595:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.030632:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983094656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320537.030652:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.030676:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983094784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320537.030700:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.030725:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983095168:0@lo:38:kworker.0 00000100:00100000:1.0:1767320537.030745:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320537.030775:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094784 00000100:00100000:1.0:1767320537.030781:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094784 00000100:00100000:1.0:1767320537.030811:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 611 00000100:00100000:1.0:1767320537.030816:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983094784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320537.030839:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320537.030875:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983094784:12345-0@lo:38:kworker.0 Request processed in 57us (173us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320537.031001:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094400 00000100:00100000:0.0:1767320537.031005:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094400 00000100:00100000:0.0:1767320537.031020:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983094656 00000100:00100000:0.0:1767320537.031022:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983094656 00000100:00100000:0.0:1767320537.031029:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095168 00000100:00100000:0.0:1767320537.031031:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095168 00000100:00100000:0.0:1767320537.031040:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2345 00000100:00100000:0.0:1767320537.031044:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983094400:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320537.031055:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320537.031082:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983094400:12345-0@lo:38:kworker.0 Request processed in 36us (486us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320537.031088:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2345 00000100:00100000:0.0:1767320537.031101:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2346 00000100:00100000:0.0:1767320537.031105:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983094656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320537.031115:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320537.031140:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983094656:12345-0@lo:38:kworker.0 Request processed in 33us (487us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320537.031145:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2346 00000100:00100000:0.0:1767320537.031153:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2347 00000100:00100000:0.0:1767320537.031157:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983095168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320537.031165:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320537.031185:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983095168:12345-0@lo:38:kworker.0 Request processed in 26us (439us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320537.031191:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2347 00000100:00080000:0.0:1767320537.031242:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320537.031247:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320537.031250:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983094400:0@lo:38:kworker.0 00000100:00080000:0.0:1767320537.031259:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320537.031262:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320537.031266:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983094656:0@lo:38:kworker.0 00000100:00080000:0.0:1767320537.031297:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320537.031301:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320537.031305:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983094784:0@lo:38:kworker.0 00000100:00080000:0.0:1767320537.031312:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320537.031316:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320537.031319:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983095168:0@lo:38:kworker.0 00000100:00080000:3.0:1767320537.031480:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320537.031532:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 611 00010000:00010000:2.0:1767320537.220614:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320538.245235:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767320539.020448:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915687040 offset 0 length 520: 4 00000100:00100000:3.0:1767320539.026341:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320539.026558:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687168 00000100:00100000:0.0:1767320539.026570:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687168 00000100:00100000:0.0:1767320539.026632:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2348 00000100:00100000:0.0:1767320539.026642:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915687168:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320539.026674:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320539.026765:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915687168:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 124us (496us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320539.026772:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2348 00000100:00100000:2.0:1767320539.029522:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320539.030217:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687296 00000100:00100000:3.0:1767320539.030232:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687296 00000100:00100000:3.0:1767320539.030389:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1775 00000100:00100000:3.0:1767320539.030402:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915687296:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320539.030522:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915687296:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 122us (1016us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320539.030532:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1775 00000100:00080000:0.0:1767320539.031047:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320539.270327:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320539.588845:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320539.588933:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320539.588962:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983095424:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320539.589070:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320539.589377:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095424 00000100:00100000:0.0:1767320539.589387:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095424 00000100:00100000:0.0:1767320539.589483:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188 00000100:00100000:0.0:1767320539.589494:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983095424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320539.589535:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320539.589593:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983095424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (523us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320539.589600:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188 00000100:00100000:0.0:1767320539.589661:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983095424:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320539.591460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320539.595488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983095296:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320539.595607:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320539.597921:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095296 00000100:00100000:0.0:1767320539.597943:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095296 00000100:00100000:0.0:1767320539.598096:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1189 00000100:00100000:0.0:1767320539.598110:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983095296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320539.598158:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320539.598634:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983095296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 526us (3025us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320539.598642:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1189 00000100:00100000:3.0:1767320539.598748:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983095296:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320539.601785:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320540.293325:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320541.124543:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320516/real 1767320516] req@ffff9346100fd880 x1853163983089024/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320541 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320541.124589:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100fd880 x1853163983089024/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320541 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320541.124615:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320541.124645:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320541.124649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983089024:0@lo:250:kworker.0 00000100:00080000:1.0:1767320541.124821:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320541.124828:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320541.124832:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320541.124838:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5765 00000100:00080000:1.0:1767320541.124843:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320541.124851:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320541.124902:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92d80 x1853163983095552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320541.124970:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320541.124975:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320541.124977:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320541.124981:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5785 00000100:00080000:1.0:1767320541.124984:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320541.124987:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320541.125005:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90380 x1853163983095808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320541.125012:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983095680:0@lo:400:kworker.0 00000100:00080000:1.0:1767320541.125021:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320541.125024:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320541.125025:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320541.125030:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5785 00000100:00080000:1.0:1767320541.125032:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320541.125035:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320541.125044:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90e00 x1853163983096064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320541.125052:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320541.125054:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320541.125056:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320541.125058:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5785 00000100:00080000:1.0:1767320541.125061:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320541.125064:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320541.125074:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91f80 x1853163983096192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320541.125095:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320541.125125:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320541.125128:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320541.125130:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320541.125133:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5785 00000100:00080000:1.0:1767320541.125136:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320541.125140:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320541.125151:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91180 x1853163983096576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320541.125168:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320541.125175:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983095936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.125200:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320541.125232:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095680 00000100:00100000:2.0:1767320541.125242:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095680 00000100:00100000:1.0:1767320541.125336:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983096320:0@lo:400:kworker.0 00000100:00100000:1.0:1767320541.125376:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320541.125409:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320541.125414:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983096448:0@lo:400:kworker.0 00000100:00100000:1.0:1767320541.125433:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320541.125985:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095936 00000100:00100000:2.0:1767320541.125990:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095936 00000100:00100000:3.0:1767320541.125993:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776 00000100:00100000:2.0:1767320541.126013:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1777 00000100:00100000:3.0:1767320541.126018:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983095680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.126019:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983095936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320541.126136:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096320 00000100:00100000:0.0:1767320541.126140:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096320 00000100:00100000:0.0:1767320541.126199:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096448 00000100:00100000:0.0:1767320541.126212:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096448 00000100:00100000:0.0:1767320541.126236:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097 00000100:00100000:0.0:1767320541.126241:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983096320:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320541.126299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320541.126304:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320541.126307:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320541.126327:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983095552:0@lo:250:kworker.0 00000100:00100000:2.0:1767320541.131958:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983095936:12345-0@lo:400:kworker.0 Request processed in 5924us (6742us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.131976:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1777 00000100:00100000:2.0:1767320541.132111:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098 00000100:00100000:2.0:1767320541.132134:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983096448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.132266:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983096448:12345-0@lo:400:kworker.0 Request processed in 146us (6831us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320541.132294:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098 00010000:00080000:2.0:1767320541.132358:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d93100 x1853163983095936/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320557 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320541.132376:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983095936:0@lo:400:kworker.0 00010000:00080000:2.0:1767320541.132404:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d93800 x1853163983095680/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320557 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320541.132413:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983095680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320541.132494:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983096448:0@lo:400:kworker.0 00000100:00100000:3.0:1767320541.144500:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983095680:12345-0@lo:400:kworker.0 Request processed in 18476us (19382us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320541.144527:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776 00000100:00100000:0.0:1767320541.144918:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983096320:12345-0@lo:400:kworker.0 Request processed in 18672us (19537us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320541.144930:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097 00000100:00100000:3.0:1767320541.145131:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983096320:0@lo:400:kworker.0 00000400:00000100:1.0:1767320541.146059:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983095552 offset 0 length 520: 4 00000100:00100000:1.0:1767320541.146117:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983095808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320541.146172:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320541.146202:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983096064:0@lo:38:kworker.0 00000100:00100000:1.0:1767320541.146230:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320541.146253:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983096192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320541.146304:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320541.146334:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983096576:0@lo:38:kworker.0 00000100:00100000:1.0:1767320541.146358:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320541.146626:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096192 00000100:00100000:0.0:1767320541.146636:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096192 00000100:00100000:0.0:1767320541.146682:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 612 00000100:00100000:0.0:1767320541.146690:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93450725f050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983096192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320541.146720:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320541.150030:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983095808 00000100:00100000:2.0:1767320541.150058:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983095808 00000100:00100000:2.0:1767320541.150170:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096064 00000100:00100000:2.0:1767320541.150174:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096064 00000100:00100000:2.0:1767320541.150192:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096576 00000100:00100000:2.0:1767320541.150196:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096576 00000100:00100000:2.0:1767320541.150218:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2349 00000100:00100000:2.0:1767320541.150230:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983095808:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320541.151042:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320541.160215:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93450725f050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983096192:12345-0@lo:38:kworker.0 Request processed in 13497us (13883us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320541.160238:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 612 00000100:00100000:1.0:1767320541.162089:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2350 00000100:00100000:1.0:1767320541.162107:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983096064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320541.162143:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320541.162251:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983096064:12345-0@lo:38:kworker.0 Request processed in 146us (16019us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320541.162258:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2350 00000100:00100000:1.0:1767320541.162306:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2351 00000100:00100000:1.0:1767320541.162311:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983096576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320541.162320:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320541.162345:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983096576:12345-0@lo:38:kworker.0 Request processed in 33us (15986us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320541.162350:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2351 00000100:00080000:1.0:1767320541.162409:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320541.162414:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320541.162418:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983096576:0@lo:38:kworker.0 00000100:00080000:1.0:1767320541.162454:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320541.162458:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320541.163064:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983096064:0@lo:38:kworker.0 00000100:00080000:1.0:1767320541.163087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320541.163092:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320541.163094:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983096192:0@lo:38:kworker.0 00000100:00100000:2.0:1767320541.163426:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983095808:12345-0@lo:38:kworker.0 Request processed in 13194us (17250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320541.163437:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2349 00000100:00080000:1.0:1767320541.164066:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320541.164073:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320541.164076:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983095808:0@lo:38:kworker.0 00010000:00010000:2.0:1767320541.321346:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320542.341911:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320543.365431:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320544.146978:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320544.147036:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320544.147126:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687552 00000100:00100000:0.0:1767320544.147140:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687552 00000100:00100000:0.0:1767320544.147233:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687424 00000100:00100000:0.0:1767320544.147236:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687424 00000100:00100000:0.0:1767320544.147259:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1778 00000100:00100000:0.0:1767320544.147309:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915687552:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320544.147398:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915687552:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 130us (431us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320544.147405:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1778 00000100:00100000:0.0:1767320544.147422:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2352 00000100:00100000:0.0:1767320544.147426:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915687424:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320544.147444:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320544.147463:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915687424:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 35us (427us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320544.147468:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2352 00000100:00080000:1.0:1767320544.148025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320544.390779:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320544.708932:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983096832:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320544.709029:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320544.709093:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320544.709100:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983096704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320544.709137:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320544.709339:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096832 00000100:00100000:1.0:1767320544.709349:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096832 00000100:00100000:1.0:1767320544.709416:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096704 00000100:00100000:1.0:1767320544.709433:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096704 00000100:00100000:1.0:1767320544.709452:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1190 00000100:00100000:1.0:1767320544.709460:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983096832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320544.709489:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320544.709546:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983096832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (517us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320544.709555:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1190 00000100:00100000:1.0:1767320544.709573:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191 00000100:00100000:1.0:1767320544.709579:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983096704:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320544.709597:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320544.709627:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983096704:12345-0@lo:13:osp-pre-0-1.0 Request processed in 48us (488us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320544.709632:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191 00000100:00080000:1.0:1767320544.709655:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320544.709659:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320544.710389:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983096704:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320544.710405:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983096832:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320545.417960:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320546.244892:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320546.244903:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320546.244977:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320546.244981:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320546.244986:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320546.244994:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5789 00000100:00080000:3.0:1767320546.244999:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320546.245006:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320546.245028:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016680 x1853163983097088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320546.245062:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320546.245065:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320546.245075:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320546.245080:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5789 00000100:00080000:3.0:1767320546.245083:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320546.245087:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320546.245099:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163983097344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320546.245108:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320546.245111:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320546.245114:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320546.245117:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5789 00000100:00080000:3.0:1767320546.245119:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320546.245124:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320546.245136:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb480 x1853163983097472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320546.245170:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320546.245174:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320546.245176:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320546.245179:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5789 00000100:00080000:3.0:1767320546.245181:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320546.245184:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320546.245194:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9f80 x1853163983097856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320546.245222:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983097728:0@lo:400:kworker.0 00000100:00100000:3.0:1767320546.245323:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320546.245362:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320546.245368:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983097600:0@lo:400:kworker.0 00000100:00100000:3.0:1767320546.245390:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320546.245487:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983096960:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245519:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320546.245546:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320546.245551:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983097216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245571:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320546.245605:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097728 00000100:00100000:2.0:1767320546.245612:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097728 00000100:00100000:2.0:1767320546.245662:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097600 00000100:00100000:2.0:1767320546.245666:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097600 00000100:00100000:2.0:1767320546.245686:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099 00000100:00100000:2.0:1767320546.245694:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983097728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245744:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983097728:12345-0@lo:400:kworker.0 Request processed in 51us (421us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320546.245751:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099 00000100:00100000:2.0:1767320546.245766:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100 00000100:00100000:2.0:1767320546.245772:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983097600:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245798:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983097600:12345-0@lo:400:kworker.0 Request processed in 27us (408us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320546.245803:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100 00000100:00100000:2.0:1767320546.245821:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983096960 00000100:00100000:2.0:1767320546.245824:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983096960 00000100:00100000:2.0:1767320546.245848:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097216 00000100:00100000:2.0:1767320546.245850:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097216 00000100:00100000:2.0:1767320546.245865:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1779 00000100:00100000:2.0:1767320546.245870:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983096960:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245902:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983096960:12345-0@lo:400:kworker.0 Request processed in 31us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320546.245908:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1779 00000100:00100000:2.0:1767320546.245919:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780 00000100:00100000:2.0:1767320546.245925:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983097216:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.245950:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983097216:12345-0@lo:400:kworker.0 Request processed in 26us (378us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320546.245955:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780 00010000:00080000:2.0:1767320546.245993:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb800 x1853163983097216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320562 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320546.246005:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983097216:0@lo:400:kworker.0 00010000:00080000:2.0:1767320546.246018:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014a80 x1853163983096960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320562 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320546.246026:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983096960:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.246056:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983097600:0@lo:400:kworker.0 00000100:00100000:2.0:1767320546.246064:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983097728:0@lo:400:kworker.0 00000100:00080000:1.0:1767320546.246458:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320546.246471:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320546.246473:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320546.246486:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983097088:0@lo:38:kworker.0 00000100:00100000:1.0:1767320546.246520:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320546.246550:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983097344:0@lo:38:kworker.0 00000100:00100000:1.0:1767320546.246576:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320546.246601:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983097472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320546.246685:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320546.246703:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983097856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320546.246724:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320546.246747:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097088 00000100:00100000:1.0:1767320546.246752:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097088 00000100:00100000:1.0:1767320546.246769:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097344 00000100:00100000:1.0:1767320546.246771:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097344 00000100:00100000:1.0:1767320546.246779:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097856 00000100:00100000:1.0:1767320546.246781:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097856 00000100:00100000:1.0:1767320546.246791:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2353 00000100:00100000:1.0:1767320546.246797:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320546.246816:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320546.246845:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097088:12345-0@lo:38:kworker.0 Request processed in 48us (325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320546.246851:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2353 00000100:00100000:1.0:1767320546.246863:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2354 00000100:00100000:1.0:1767320546.246866:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320546.246875:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320546.246896:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097344:12345-0@lo:38:kworker.0 Request processed in 28us (319us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320546.246901:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2354 00000100:00100000:1.0:1767320546.246911:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2355 00000100:00100000:1.0:1767320546.246914:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320546.246923:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320546.246943:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983097856:12345-0@lo:38:kworker.0 Request processed in 27us (219us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320546.246949:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2355 00000100:00080000:1.0:1767320546.246974:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320546.246979:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320546.246982:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983097856:0@lo:38:kworker.0 00000100:00080000:1.0:1767320546.247000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320546.247004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320546.247006:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983097088:0@lo:38:kworker.0 00000100:00080000:1.0:1767320546.247013:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320546.247016:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320546.247018:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983097344:0@lo:38:kworker.0 00000100:00100000:0.0:1767320546.247511:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097472 00000100:00100000:0.0:1767320546.247518:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097472 00000100:00100000:0.0:1767320546.247557:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 613 00000100:00100000:0.0:1767320546.247563:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983097472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320546.247574:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320546.247625:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983097472:12345-0@lo:38:kworker.0 Request processed in 61us (939us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320546.247636:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 613 00000100:00080000:2.0:1767320546.247801:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320546.247806:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320546.247810:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983097472:0@lo:38:kworker.0 00010000:00010000:2.0:1767320546.437695:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320547.460526:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320548.484817:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320549.259828:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320549.259881:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320549.259998:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687808 00000100:00100000:0.0:1767320549.260009:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687808 00000100:00100000:3.0:1767320549.260015:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687680 00000100:00100000:3.0:1767320549.260022:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687680 00000100:00100000:3.0:1767320549.260089:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1781 00000100:00100000:3.0:1767320549.260101:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915687808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767320549.260114:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320549.260186:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915687808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (376us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320549.260194:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1781 00000100:00100000:3.0:1767320549.260212:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2356 00000100:00100000:3.0:1767320549.260216:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915687680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320549.260267:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320549.260334:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915687680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 116us (453us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320549.260341:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2356 00010000:00010000:2.0:1767320549.508999:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320549.828686:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320549.831500:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983098112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320549.831600:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320549.831655:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983097984:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320549.831679:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320549.831942:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098112 00000100:00100000:1.0:1767320549.831951:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098112 00000100:00100000:1.0:1767320549.832022:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983097984 00000100:00100000:1.0:1767320549.832026:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983097984 00000100:00100000:1.0:1767320549.832046:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1192 00000100:00100000:1.0:1767320549.832057:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983098112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320549.832091:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320549.832139:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983098112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (539us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320549.832146:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1192 00000100:00100000:1.0:1767320549.832174:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193 00000100:00100000:1.0:1767320549.832182:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983097984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320549.832198:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320549.832224:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983097984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (545us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320549.832240:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193 00000100:00080000:3.0:1767320549.832442:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320549.832447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320549.832484:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983098112:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320549.832496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983097984:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320550.533035:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320551.377690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320551.377701:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320551.377774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320551.377779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320551.377782:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320551.377792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5794 00000100:00080000:2.0:1767320551.377798:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320551.377806:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320551.377833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163983098368/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320551.377874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320551.377877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320551.377879:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320551.377883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5794 00000100:00080000:2.0:1767320551.377885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320551.377889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320551.377903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580380 x1853163983098624/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320551.377911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320551.377914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320551.377916:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320551.377919:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5794 00000100:00080000:2.0:1767320551.377921:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320551.377925:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320551.377938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016680 x1853163983098752/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320551.377951:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983098240:0@lo:400:kworker.0 00000100:00080000:2.0:1767320551.377965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320551.377968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320551.377969:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320551.377972:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5794 00000100:00080000:2.0:1767320551.377974:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320551.377977:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320551.377986:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163983099136/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320551.378022:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.378052:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098240 00000100:00100000:2.0:1767320551.378058:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098240 00000100:00100000:0.0:1767320551.378064:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320551.378070:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983098496:0@lo:400:kworker.0 00000100:00100000:0.0:1767320551.378093:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320551.378158:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983098880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.378159:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983099008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320551.378189:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320551.378190:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320551.378225:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320551.378238:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983098368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.378252:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098880 00000100:00100000:3.0:1767320551.378255:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098880 00000100:00100000:1.0:1767320551.378261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.378319:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101 00000100:00100000:3.0:1767320551.378329:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983098880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.378379:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983098880:12345-0@lo:400:kworker.0 Request processed in 51us (188us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320551.378386:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101 00000100:00100000:3.0:1767320551.378416:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099008 00000100:00100000:3.0:1767320551.378420:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099008 00000100:00100000:3.0:1767320551.378455:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1102 00000100:00100000:3.0:1767320551.378462:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983099008:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.378494:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983099008:12345-0@lo:400:kworker.0 Request processed in 33us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320551.378500:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1102 00000100:00080000:3.0:1767320551.378522:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320551.378525:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320551.378554:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983099008:0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.378585:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983098880:0@lo:400:kworker.0 00000100:00100000:1.0:1767320551.408941:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983098624:0@lo:38:kworker.0 00000100:00100000:1.0:1767320551.409013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320551.409079:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098496 00000100:00100000:2.0:1767320551.409088:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098496 00000100:00100000:2.0:1767320551.409157:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098368 00000100:00100000:2.0:1767320551.409161:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098368 00000100:00100000:2.0:1767320551.409176:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1782 00000100:00100000:2.0:1767320551.409185:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983098240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.409257:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983098240:12345-0@lo:400:kworker.0 Request processed in 73us (31234us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.409265:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1782 00000100:00100000:2.0:1767320551.409298:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1783 00000100:00100000:2.0:1767320551.409305:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983098496:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320551.409335:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983098496:12345-0@lo:400:kworker.0 Request processed in 30us (31241us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320551.409341:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1783 00000100:00100000:2.0:1767320551.409353:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2357 00000100:00100000:2.0:1767320551.409357:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983098368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320551.409375:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320551.409402:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983098368:12345-0@lo:38:kworker.0 Request processed in 43us (31139us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320551.409408:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2357 00000100:00080000:3.0:1767320551.409858:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320551.409938:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604583b80 x1853163983098496/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320567 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320551.409954:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983098496:0@lo:400:kworker.0 00010000:00080000:3.0:1767320551.409967:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582680 x1853163983098240/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320567 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320551.409980:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983098240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320551.421723:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098624 00000100:00100000:1.0:1767320551.421734:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983098752:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.421736:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098624 00000100:00100000:3.0:1767320551.421778:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2358 00000100:00100000:3.0:1767320551.421785:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983098624:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320551.421791:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767320551.421811:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320551.421825:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983099136:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.421858:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983098624:12345-0@lo:38:kworker.0 Request processed in 72us (12844us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320551.421862:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320551.421866:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2358 00000100:00100000:3.0:1767320551.421880:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099136 00000100:00100000:3.0:1767320551.421883:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099136 00000100:00100000:3.0:1767320551.421896:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2359 00000100:00100000:3.0:1767320551.421900:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983099136:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320551.421904:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320551.421908:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:3.0:1767320551.421909:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320551.421912:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983098368:0@lo:38:kworker.0 00000100:00080000:1.0:1767320551.421921:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320551.421925:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320551.421928:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983098624:0@lo:38:kworker.0 00000100:00100000:3.0:1767320551.421930:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983099136:12345-0@lo:38:kworker.0 Request processed in 29us (68us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320551.421935:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320551.421936:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2359 00000100:00080000:1.0:1767320551.421938:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320551.421941:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983099136:0@lo:38:kworker.0 00000100:00100000:2.0:1767320551.422011:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983098752 00000100:00100000:2.0:1767320551.422016:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983098752 00000100:00100000:2.0:1767320551.422048:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 614 00000100:00100000:2.0:1767320551.422053:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983098752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320551.422063:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320551.443465:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983098752:12345-0@lo:38:kworker.0 Request processed in 21395us (21657us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320551.443488:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 614 00000100:00080000:1.0:1767320551.443513:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320551.443530:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320551.443535:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983098752:0@lo:38:kworker.0 00010000:00010000:1.0:1767320551.556768:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320552.584296:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320553.604510:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320554.382914:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320554.383065:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688064 00000100:00100000:3.0:1767320554.383077:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688064 00000100:00100000:3.0:1767320554.383196:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1784 00000100:00100000:3.0:1767320554.383207:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688064:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320554.383340:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688064:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 135us (437us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320554.383349:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1784 00000100:00080000:1.0:1767320554.383775:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320554.401884:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320554.401992:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915687936 00000100:00100000:2.0:1767320554.402004:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915687936 00000100:00100000:2.0:1767320554.402060:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2360 00000100:00100000:2.0:1767320554.402069:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915687936:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320554.402092:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320554.402178:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915687936:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 107us (305us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320554.402186:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2360 00010000:00010000:1.0:1767320554.628503:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320554.949594:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983099264:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320554.949667:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320554.949777:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320554.949784:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983099392:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320554.949804:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320554.952796:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099264 00000100:00100000:0.0:1767320554.952812:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099264 00000100:00100000:0.0:1767320554.952902:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099392 00000100:00100000:0.0:1767320554.952906:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099392 00000100:00100000:2.0:1767320554.952952:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1194 00000100:00100000:2.0:1767320554.952966:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983099264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320554.953001:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320554.953075:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983099264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (3407us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320554.953095:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1194 00000100:00100000:3.0:1767320554.953727:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983099264:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320554.953754:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320554.954160:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1195 00000100:00100000:0.0:1767320554.954170:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983099392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320554.954192:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320554.954252:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983099392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 83us (4446us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320554.954259:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1195 00000100:00080000:3.0:1767320554.954334:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320554.954362:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983099392:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320555.653479:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320556.489616:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320556.489627:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320556.489759:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983099520:0@lo:400:kworker.0 00000100:00100000:1.0:1767320556.489946:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320556.490070:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099520 00000100:00100000:2.0:1767320556.490079:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099520 00000100:00100000:2.0:1767320556.490168:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1785 00000100:00100000:2.0:1767320556.490178:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983099520:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320556.490244:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320556.490248:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320556.490252:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320556.490257:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5799 00000100:00080000:3.0:1767320556.490261:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320556.490266:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320556.490307:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163983099648/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320556.490374:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320556.490407:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983099648:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.490438:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.490486:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099648 00000100:00100000:0.0:1767320556.490491:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099648 00000100:00100000:0.0:1767320556.490513:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2361 00000100:00100000:0.0:1767320556.490519:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983099648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320556.490537:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.490586:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983099648:12345-0@lo:38:kworker.0 Request processed in 65us (145us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.490593:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2361 00000100:00100000:0.0:1767320556.490618:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983099776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.490641:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320556.490681:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.490686:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.490689:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983099648:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.490708:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099776 00000100:00100000:0.0:1767320556.490711:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099776 00000100:00100000:0.0:1767320556.490750:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1786 00000100:00100000:0.0:1767320556.490757:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983099776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.490789:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983099776:12345-0@lo:400:kworker.0 Request processed in 34us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.490808:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1786 00010000:00080000:0.0:1767320556.490839:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eaa00 x1853163983099776/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320572 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320556.490849:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983099776:0@lo:400:kworker.0 00000100:00080000:0.0:1767320556.490862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320556.496483:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983099520:12345-0@lo:400:kworker.0 Request processed in 6294us (6524us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320556.496500:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1785 00000100:00080000:3.0:1767320556.498457:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320556.498467:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320556.498471:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320556.498478:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5799 00000100:00080000:3.0:1767320556.498482:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320556.498489:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320556.498530:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb100 x1853163983099904/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320556.498550:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320556.498553:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320556.498564:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320556.498567:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5799 00000100:00080000:3.0:1767320556.498570:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320556.498573:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320556.498587:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eaa00 x1853163983100032/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320556.498627:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320556.498631:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320556.498633:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320556.498636:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5799 00000100:00080000:3.0:1767320556.498638:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320556.498642:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320556.498652:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163983100416/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320556.498945:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983099904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.499022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.499080:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983100032:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.499104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.499136:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983100416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320556.499167:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.499212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983099904 00000100:00100000:0.0:1767320556.499219:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983099904 00000100:00100000:0.0:1767320556.499248:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100416 00000100:00100000:0.0:1767320556.499251:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100416 00000100:00100000:0.0:1767320556.499268:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2362 00000100:00100000:0.0:1767320556.499295:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983099904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320556.499314:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.499359:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983099904:12345-0@lo:38:kworker.0 Request processed in 64us (337us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.499366:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2362 00000100:00100000:0.0:1767320556.499382:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2363 00000100:00100000:0.0:1767320556.499386:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983100416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320556.499394:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320556.499417:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983100416:12345-0@lo:38:kworker.0 Request processed in 30us (250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320556.499423:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2363 00000100:00080000:0.0:1767320556.499461:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.499466:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.499469:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983099904:0@lo:38:kworker.0 00000100:00080000:0.0:1767320556.499478:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320556.499481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320556.499484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983100416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320556.499861:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983100288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320556.499895:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320556.499932:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320556.499938:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983100160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320556.499958:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320556.500676:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100288 00000100:00100000:0.0:1767320556.500683:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100288 00000100:00100000:0.0:1767320556.500750:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100160 00000100:00100000:0.0:1767320556.500753:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100160 00000100:00100000:0.0:1767320556.500772:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103 00000100:00100000:0.0:1767320556.500778:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983100288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.500975:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983100288:12345-0@lo:400:kworker.0 Request processed in 196us (1079us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.500982:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103 00000100:00100000:0.0:1767320556.500997:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104 00000100:00100000:0.0:1767320556.501003:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983100160:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320556.501031:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983100160:12345-0@lo:400:kworker.0 Request processed in 28us (1071us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320556.501037:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104 00000100:00100000:3.0:1767320556.501154:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100032 00000100:00100000:3.0:1767320556.501160:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100032 00000100:00100000:3.0:1767320556.501190:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 615 00000100:00100000:3.0:1767320556.501195:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983100032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320556.501215:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320556.501257:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983100032:12345-0@lo:38:kworker.0 Request processed in 61us (2152us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320556.501263:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 615 00000100:00080000:3.0:1767320556.501320:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320556.501325:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320556.501329:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983100032:0@lo:38:kworker.0 00000100:00080000:2.0:1767320556.501474:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320556.501479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320556.501499:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983100160:0@lo:400:kworker.0 00000100:00100000:2.0:1767320556.501507:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983100288:0@lo:400:kworker.0 00010000:00080000:1.0:1767320556.502888:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9180 x1853163983099520/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320572 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320556.502905:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983099520:0@lo:400:kworker.0 00010000:00010000:2.0:1767320556.678415:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320557.701389:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320558.724650:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320559.501805:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320559.501838:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320559.501903:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688320 00000100:00100000:1.0:1767320559.501917:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688320 00000100:00100000:2.0:1767320559.501924:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688192 00000100:00100000:2.0:1767320559.501928:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688192 00000100:00100000:2.0:1767320559.501966:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1787 00000100:00100000:1.0:1767320559.501976:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2364 00000100:00100000:2.0:1767320559.501976:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688320:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320559.501982:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915688192:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320559.502002:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320559.502048:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915688192:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 65us (207us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767320559.502063:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320559.502070:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2364 00000100:00100000:2.0:1767320559.507033:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688320:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 5046us (5223us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320559.507058:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1787 00010000:00010000:2.0:1767320559.750525:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320560.068849:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983100544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320560.068927:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320560.069070:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100544 00000100:00100000:0.0:1767320560.069080:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100544 00000100:00100000:0.0:1767320560.069226:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196 00000100:00100000:0.0:1767320560.069237:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983100544:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320560.069265:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320560.069335:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983100544:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (407us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320560.069342:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196 00000100:00080000:3.0:1767320560.069537:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320560.069574:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983100544:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320560.069591:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320560.069747:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983100672:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320560.069777:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320560.069882:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100672 00000100:00100000:2.0:1767320560.069889:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100672 00000100:00100000:2.0:1767320560.069938:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197 00000100:00100000:2.0:1767320560.069946:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983100672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320560.069965:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320560.070002:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983100672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (223us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320560.070008:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197 00000100:00080000:3.0:1767320560.070582:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320560.070675:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983100672:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320560.773711:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320561.604419:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320561.604431:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320561.604488:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320561.604493:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320561.604497:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320561.604505:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5805 00000100:00080000:3.0:1767320561.604511:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320561.604518:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320561.604543:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9180 x1853163983100928/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320561.604579:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320561.604582:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320561.604584:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320561.604588:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5805 00000100:00080000:3.0:1767320561.604590:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320561.604594:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320561.604605:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163983101184/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320561.604615:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320561.604618:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320561.604620:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320561.604623:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5805 00000100:00080000:3.0:1767320561.604626:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320561.604628:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320561.604642:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8380 x1853163983101312/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320561.604674:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320561.604677:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320561.604679:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320561.604683:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5805 00000100:00080000:3.0:1767320561.604686:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320561.604689:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320561.604699:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163983101696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320561.604726:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983101568:0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.604791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.604856:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320561.604862:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983101440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.604885:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.604916:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101568 00000100:00100000:3.0:1767320561.604923:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101568 00000100:00100000:3.0:1767320561.604963:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101440 00000100:00100000:3.0:1767320561.604966:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101440 00000100:00100000:3.0:1767320561.604988:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1105 00000100:00100000:3.0:1767320561.604997:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983101568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.605058:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983101568:12345-0@lo:400:kworker.0 Request processed in 61us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320561.605065:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1105 00000100:00100000:3.0:1767320561.605080:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106 00000100:00100000:3.0:1767320561.605086:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983101440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.605113:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983101440:12345-0@lo:400:kworker.0 Request processed in 27us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320561.605120:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106 00000100:00100000:3.0:1767320561.605165:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983101440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.605175:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983101568:0@lo:400:kworker.0 00000100:00100000:1.0:1767320561.605302:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983100928:0@lo:38:kworker.0 00000100:00100000:1.0:1767320561.605338:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.605373:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983101184:0@lo:38:kworker.0 00000100:00100000:1.0:1767320561.605395:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.605412:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983101312:0@lo:38:kworker.0 00000100:00100000:1.0:1767320561.605435:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.605457:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983101696:0@lo:38:kworker.0 00000100:00100000:1.0:1767320561.605476:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320561.605503:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100928 00000100:00100000:1.0:1767320561.605508:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100928 00000100:00100000:1.0:1767320561.605527:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101184 00000100:00100000:1.0:1767320561.605530:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101184 00000100:00100000:1.0:1767320561.605542:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101696 00000100:00100000:1.0:1767320561.605545:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101696 00000100:00100000:1.0:1767320561.605554:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2365 00000100:00100000:1.0:1767320561.605559:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983100928:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320561.605577:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.605612:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983100928:12345-0@lo:38:kworker.0 Request processed in 51us (273us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320561.605618:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2365 00000100:00100000:1.0:1767320561.605632:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2366 00000100:00100000:1.0:1767320561.605637:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983101184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320561.605646:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.605667:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983101184:12345-0@lo:38:kworker.0 Request processed in 29us (271us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320561.605673:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2366 00000100:00100000:1.0:1767320561.605684:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2367 00000100:00100000:1.0:1767320561.605687:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983101696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320561.605695:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.605717:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983101696:12345-0@lo:38:kworker.0 Request processed in 28us (240us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320561.605723:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2367 00000100:00100000:1.0:1767320561.605740:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101312 00000100:00100000:1.0:1767320561.605743:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101312 00000100:00100000:1.0:1767320561.605762:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 616 00000100:00100000:1.0:1767320561.605766:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983101312:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320561.605775:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320561.605800:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983101312:12345-0@lo:38:kworker.0 Request processed in 32us (363us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320561.605805:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 616 00000100:00080000:1.0:1767320561.605842:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320561.605846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320561.605850:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983100928:0@lo:38:kworker.0 00000100:00080000:1.0:1767320561.605858:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320561.605861:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320561.605864:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983101184:0@lo:38:kworker.0 00000100:00080000:1.0:1767320561.605872:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320561.605875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320561.605878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983101312:0@lo:38:kworker.0 00000100:00080000:1.0:1767320561.605885:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320561.605888:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320561.605890:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983101696:0@lo:38:kworker.0 00000100:00100000:0.0:1767320561.606055:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983100800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.606087:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320561.606115:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320561.606121:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983101056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320561.606142:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320561.606214:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983100800 00000100:00100000:3.0:1767320561.606219:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983100800 00000100:00100000:3.0:1767320561.606263:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101056 00000100:00100000:3.0:1767320561.606266:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101056 00000100:00100000:3.0:1767320561.606310:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788 00000100:00100000:3.0:1767320561.606317:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983100800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.606356:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983100800:12345-0@lo:400:kworker.0 Request processed in 39us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320561.606361:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788 00000100:00100000:3.0:1767320561.606373:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1789 00000100:00100000:3.0:1767320561.606379:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983101056:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320561.606404:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983101056:12345-0@lo:400:kworker.0 Request processed in 26us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320561.606409:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1789 00000100:00080000:0.0:1767320561.606464:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320561.606487:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb480 x1853163983101056/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320577 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320561.606499:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983101056:0@lo:400:kworker.0 00010000:00080000:0.0:1767320561.606508:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb800 x1853163983100800/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320577 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320561.606516:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983100800:0@lo:400:kworker.0 00010000:00010000:2.0:1767320561.799236:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320562.820623:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320563.844505:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320564.619584:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320564.619676:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688704 00000100:00100000:1.0:1767320564.619685:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688704 00000100:00080000:0.0:1767320564.619784:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320564.619821:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1790 00000100:00100000:1.0:1767320564.619830:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688704:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320564.619900:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688704:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 71us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320564.619907:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1790 00000400:00000100:1.0:1767320564.621354:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915688448 offset 0 length 520: 4 00000100:00100000:1.0:1767320564.621453:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320564.621615:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688576 00000100:00100000:2.0:1767320564.621623:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688576 00000100:00100000:2.0:1767320564.621668:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2368 00000100:00100000:2.0:1767320564.621676:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915688576:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320564.621698:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320564.622223:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915688576:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 545us (767us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320564.622232:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2368 00010000:00010000:2.0:1767320564.870518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320565.188686:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320565.188711:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983101824:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320565.188785:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320565.189106:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320565.189117:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983101952:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320565.189164:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320565.189336:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101824 00000100:00100000:1.0:1767320565.189344:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101824 00000100:00100000:1.0:1767320565.189405:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983101952 00000100:00100000:1.0:1767320565.189408:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983101952 00000100:00100000:1.0:1767320565.189427:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1198 00000100:00100000:1.0:1767320565.189436:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983101824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320565.189467:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320565.189511:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983101824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (725us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320565.189518:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1198 00000100:00100000:1.0:1767320565.189535:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767320565.189542:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983101952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320565.189558:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320565.189596:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983101824:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320565.189660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320565.189666:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320565.189734:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983101952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 192us (569us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320565.189741:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199 00000100:00100000:2.0:1767320565.189811:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983101952:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320565.895475:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320566.725174:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320541/real 1767320541] req@ffff934639d92d80 x1853163983095552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320566 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320566.725225:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934639d92d80 x1853163983095552/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320566 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320566.725245:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320566.725267:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320566.725299:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983095552:0@lo:250:kworker.0 00000100:00080000:3.0:1767320566.726127:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320566.726134:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320566.726137:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320566.726143:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5789 00000100:00080000:3.0:1767320566.726147:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320566.726154:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320566.726189:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163983102080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320566.726262:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320566.726266:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320566.726268:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320566.726292:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5810 00000100:00080000:3.0:1767320566.726294:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320566.726298:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320566.726312:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb480 x1853163983102336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320566.726342:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320566.726345:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320566.726347:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320566.726350:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5810 00000100:00080000:3.0:1767320566.726352:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320566.726356:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320566.726365:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9c00 x1853163983102592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320566.726373:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320566.726377:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320566.726378:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320566.726382:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5810 00000100:00080000:3.0:1767320566.726384:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320566.726387:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320566.726397:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9180 x1853163983102720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320566.726567:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983102464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320566.726637:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320566.726698:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320566.726705:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983102208:0@lo:400:kworker.0 00000100:00100000:2.0:1767320566.726729:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320566.726765:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102464 00000100:00100000:2.0:1767320566.726773:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102464 00000100:00100000:2.0:1767320566.726819:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102208 00000100:00100000:2.0:1767320566.726822:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102208 00000100:00100000:2.0:1767320566.726846:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1791 00000100:00100000:2.0:1767320566.726853:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983102464:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320566.726899:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983102464:12345-0@lo:400:kworker.0 Request processed in 46us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320566.726906:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1791 00000100:00100000:2.0:1767320566.726955:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792 00000100:00100000:2.0:1767320566.726962:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983102208:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320566.726989:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983102208:12345-0@lo:400:kworker.0 Request processed in 27us (259us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320566.726995:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792 00000100:00080000:2.0:1767320566.727011:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320566.727035:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395eb800 x1853163983102208/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320582 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320566.727048:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983102208:0@lo:400:kworker.0 00010000:00080000:2.0:1767320566.727058:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8380 x1853163983102464/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320582 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320566.727066:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983102464:0@lo:400:kworker.0 00000100:00080000:3.0:1767320566.727221:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320566.727225:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320566.727227:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320566.727230:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5810 00000100:00080000:3.0:1767320566.727233:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320566.727237:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320566.727251:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395eb800 x1853163983103104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320566.727299:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983102848:0@lo:400:kworker.0 00000100:00100000:3.0:1767320566.727344:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320566.727375:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320566.727381:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983102976:0@lo:400:kworker.0 00000100:00100000:3.0:1767320566.727400:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320566.727566:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102848 00000100:00100000:0.0:1767320566.727574:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102848 00000100:00100000:0.0:1767320566.727605:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102976 00000100:00100000:0.0:1767320566.727609:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102976 00000100:00100000:0.0:1767320566.727627:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107 00000100:00100000:0.0:1767320566.727633:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983102848:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320566.727670:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983102848:12345-0@lo:400:kworker.0 Request processed in 36us (325us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320566.727675:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107 00000100:00100000:0.0:1767320566.727688:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108 00000100:00100000:0.0:1767320566.727693:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983102976:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320566.727715:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983102976:12345-0@lo:400:kworker.0 Request processed in 22us (314us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320566.727720:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108 00000100:00100000:3.0:1767320566.727850:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983102976:0@lo:400:kworker.0 00000100:00100000:3.0:1767320566.727859:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983102848:0@lo:400:kworker.0 00000100:00100000:1.0:1767320566.734165:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983102080:0@lo:250:kworker.0 00000400:00000100:1.0:1767320566.734242:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983102080 offset 0 length 520: 4 00000100:00100000:1.0:1767320566.734324:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983102336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320566.734361:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320566.734396:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983102592:0@lo:38:kworker.0 00000100:00100000:1.0:1767320566.734419:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320566.734443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983102720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320566.734467:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320566.734493:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983103104:0@lo:38:kworker.0 00000100:00100000:1.0:1767320566.734513:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320566.734562:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102336 00000100:00100000:1.0:1767320566.734569:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102336 00000100:00100000:1.0:1767320566.734597:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102592 00000100:00100000:1.0:1767320566.734600:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102592 00000100:00100000:1.0:1767320566.734612:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103104 00000100:00100000:1.0:1767320566.734614:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103104 00000100:00100000:1.0:1767320566.734624:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2369 00000100:00100000:1.0:1767320566.734631:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983102336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320566.734651:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320566.734690:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983102336:12345-0@lo:38:kworker.0 Request processed in 58us (328us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320566.734697:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2369 00000100:00100000:1.0:1767320566.734713:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2370 00000100:00100000:1.0:1767320566.734717:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983102592:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320566.734726:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320566.734749:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983102592:12345-0@lo:38:kworker.0 Request processed in 31us (329us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320566.734755:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2370 00000100:00100000:1.0:1767320566.734766:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2371 00000100:00100000:1.0:1767320566.734770:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983103104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320566.734779:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320566.734799:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983103104:12345-0@lo:38:kworker.0 Request processed in 28us (285us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320566.734805:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2371 00000100:00080000:1.0:1767320566.734843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320566.734847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320566.734851:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983102336:0@lo:38:kworker.0 00000100:00080000:1.0:1767320566.734859:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320566.734862:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320566.734865:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983102592:0@lo:38:kworker.0 00000100:00080000:1.0:1767320566.734872:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320566.734875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320566.734878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983103104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320566.735040:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983102720 00000100:00100000:0.0:1767320566.735045:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983102720 00000100:00100000:0.0:1767320566.735074:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 617 00000100:00100000:0.0:1767320566.735079:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983102720:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320566.735089:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320566.735124:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983102720:12345-0@lo:38:kworker.0 Request processed in 44us (655us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320566.735130:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 617 00000100:00080000:1.0:1767320566.735241:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320566.735246:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320566.735249:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983102720:0@lo:38:kworker.0 00010000:00010000:1.0:1767320566.916576:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320567.940671:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320568.964951:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320569.739785:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320569.739843:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320569.740109:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688960 00000100:00100000:2.0:1767320569.740126:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688960 00000100:00100000:1.0:1767320569.740321:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915688832 00000100:00100000:1.0:1767320569.740327:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915688832 00000100:00100000:2.0:1767320569.740332:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1793 00000100:00100000:2.0:1767320569.740345:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320569.740362:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2372 00000100:00100000:1.0:1767320569.740369:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915688832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320569.740390:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320569.740829:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915688960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 485us (1057us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320569.740837:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1793 00000100:00080000:3.0:1767320569.741069:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320569.744432:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915688832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 4048us (4574us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320569.744448:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2372 00010000:00010000:1.0:1767320569.989798:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320570.308667:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983103232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320570.308752:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320570.308842:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320570.308850:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983103360:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320570.308898:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320570.316029:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103232 00000100:00100000:3.0:1767320570.316046:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103232 00000100:00100000:3.0:1767320570.316182:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103360 00000100:00100000:3.0:1767320570.316186:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103360 00000100:00100000:1.0:1767320570.317902:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200 00000100:00100000:1.0:1767320570.317928:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983103232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320570.317969:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320570.318092:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983103232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 176us (9339us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320570.318101:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200 00000100:00100000:1.0:1767320570.318180:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983103232:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320570.321571:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320570.321584:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320570.322935:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1201 00000100:00100000:3.0:1767320570.322955:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983103360:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320570.323006:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320570.323101:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983103360:12345-0@lo:13:osp-pre-0-1.0 Request processed in 155us (14202us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320570.323109:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1201 00000100:00100000:1.0:1767320570.323198:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983103360:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320571.014407:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320571.846677:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320571.846687:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767320571.846804:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983103488:0@lo:400:kworker.0 00000100:00100000:3.0:1767320571.846916:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320571.846993:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320571.846998:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320571.847001:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320571.847007:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5815 00000100:00080000:2.0:1767320571.847011:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320571.847026:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320571.847051:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960a80 x1853163983103616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320571.847088:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103488 00000100:00100000:0.0:1767320571.847130:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983103616:0@lo:38:kworker.0 00000100:00100000:0.0:1767320571.847167:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320571.847196:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320571.847206:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983103744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320571.847234:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103616 00000100:00100000:3.0:1767320571.847235:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320571.847243:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103616 00000100:00080000:2.0:1767320571.847309:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320571.847313:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320571.847316:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767320571.847321:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103744 00000100:00080000:2.0:1767320571.847321:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5815 00000100:00080000:2.0:1767320571.847326:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320571.847327:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103744 00000100:00080000:2.0:1767320571.847331:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320571.847347:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960000 x1853163983103872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320571.847367:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320571.847370:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320571.847372:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320571.847375:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5815 00000100:00080000:2.0:1767320571.847377:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320571.847381:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320571.847392:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103488 00000100:00100000:2.0:1767320571.847396:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163983104000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320571.847457:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320571.847461:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320571.847462:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320571.847466:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5815 00000100:00080000:2.0:1767320571.847468:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320571.847471:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320571.847483:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963480 x1853163983104384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320571.847790:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983104256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320571.847836:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320571.847860:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320571.847865:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983104128:0@lo:400:kworker.0 00000100:00100000:2.0:1767320571.847884:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320571.847909:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104256 00000100:00100000:2.0:1767320571.847913:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104256 00000100:00100000:2.0:1767320571.847943:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104128 00000100:00100000:2.0:1767320571.847947:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104128 00000100:00100000:2.0:1767320571.847966:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109 00000100:00100000:2.0:1767320571.847974:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604771450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983104256:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320571.848131:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604771450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983104256:12345-0@lo:400:kworker.0 Request processed in 156us (293us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320571.848139:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109 00000100:00100000:2.0:1767320571.848363:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110 00000100:00100000:2.0:1767320571.848371:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983104128:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320571.848400:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983104128:12345-0@lo:400:kworker.0 Request processed in 30us (514us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320571.848406:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110 00000100:00100000:2.0:1767320571.848436:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983103872:0@lo:38:kworker.0 00000100:00100000:2.0:1767320571.848616:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320571.850366:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983104000:0@lo:38:kworker.0 00000100:00100000:2.0:1767320571.850461:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320571.850491:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983104384:0@lo:38:kworker.0 00000100:00100000:2.0:1767320571.850632:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320571.850937:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983103872 00000100:00100000:3.0:1767320571.850946:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983103872 00000100:00100000:3.0:1767320571.850983:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1794 00000100:00100000:3.0:1767320571.850995:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983103744:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320571.851039:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1795 00000100:00100000:0.0:1767320571.851051:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983103488:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320571.851103:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983103488:12345-0@lo:400:kworker.0 Request processed in 55us (4186us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320571.851111:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1795 00000100:00100000:0.0:1767320571.851136:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2373 00000100:00100000:3.0:1767320571.851141:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983103744:12345-0@lo:400:kworker.0 Request processed in 149us (3904us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320571.851145:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983103616:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320571.851154:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1794 00010000:00020000:0.0:1767320571.851172:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320571.851176:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2374 00000100:00100000:3.0:1767320571.851181:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983103872:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320571.851192:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320571.851208:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983103616:12345-0@lo:38:kworker.0 Request processed in 63us (4039us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320571.851217:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2373 00000100:00100000:3.0:1767320571.851226:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983103872:12345-0@lo:38:kworker.0 Request processed in 41us (2608us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320571.851235:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2374 00000100:00080000:0.0:1767320571.851257:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320571.851301:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962a00 x1853163983103744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320587 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320571.851316:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983103744:0@lo:400:kworker.0 00010000:00080000:3.0:1767320571.851334:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962680 x1853163983103488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320587 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320571.851344:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983103488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320571.851355:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983104128:0@lo:400:kworker.0 00000100:00100000:0.0:1767320571.851371:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983104256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320571.851387:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104000 00000100:00100000:3.0:1767320571.851393:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104000 00000100:00100000:3.0:1767320571.851415:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 618 00000100:00100000:3.0:1767320571.851421:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983104000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320571.851790:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320571.851821:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983104000:12345-0@lo:38:kworker.0 Request processed in 400us (1361us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320571.851839:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 618 00000100:00080000:2.0:1767320571.853137:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320571.853138:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104384 00000100:00100000:1.0:1767320571.853148:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104384 00000100:00080000:2.0:1767320571.853149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320571.853156:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983103616:0@lo:38:kworker.0 00000100:00080000:2.0:1767320571.853189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320571.853194:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320571.853197:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2375 00000100:00100000:2.0:1767320571.853201:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983103872:0@lo:38:kworker.0 00000100:00100000:1.0:1767320571.853208:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983104384:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767320571.853214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320571.853219:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320571.853223:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983104000:0@lo:38:kworker.0 00010000:00020000:1.0:1767320571.853228:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320571.853304:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983104384:12345-0@lo:38:kworker.0 Request processed in 96us (2670us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320571.853311:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2375 00000100:00080000:2.0:1767320571.853313:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320571.853320:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320571.853325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983104384:0@lo:38:kworker.0 00010000:00010000:2.0:1767320572.036700:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320573.060645:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320574.084475:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320574.863442:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320574.863789:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320574.863903:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689216 00000100:00100000:0.0:1767320574.863914:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689216 00000100:00100000:1.0:1767320574.863932:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689088 00000100:00100000:1.0:1767320574.863939:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689088 00000100:00080000:2.0:1767320574.864020:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320574.864064:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1796 00000100:00100000:1.0:1767320574.864076:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915689216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320574.864195:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915689216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 121us (762us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320574.864204:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1796 00000100:00100000:1.0:1767320574.864220:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2376 00000100:00100000:1.0:1767320574.864223:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915689088:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320574.864240:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320574.864260:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915689088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 36us (473us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320574.864263:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2376 00010000:00010000:2.0:1767320575.108519:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320575.428535:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983104512:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320575.428607:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320575.428914:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983104640:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320575.428943:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320575.429872:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104512 00000100:00100000:2.0:1767320575.429882:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104512 00000100:00100000:2.0:1767320575.429932:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104640 00000100:00100000:2.0:1767320575.429935:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104640 00000100:00100000:2.0:1767320575.429950:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1202 00000100:00100000:2.0:1767320575.429958:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983104512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320575.429987:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767320575.430063:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320575.430067:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320575.430083:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203 00000100:00100000:1.0:1767320575.430091:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983104640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320575.430108:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320575.430148:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983104640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (1204us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320575.430155:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203 00000100:00100000:1.0:1767320575.430206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983104640:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320575.434021:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983104512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4051us (5400us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320575.434036:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1202 00000100:00100000:3.0:1767320575.434466:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983104512:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320576.141471:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320576.964905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320576.964915:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320576.964986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320576.964990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320576.964994:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320576.965002:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320576.965006:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320576.965013:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320576.965046:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163983104896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320576.965075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320576.965079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320576.965081:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320576.965084:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320576.965087:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320576.965091:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320576.965242:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983104768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320576.965307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983104896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320576.965326:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320576.965373:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320576.965376:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104768 00000100:00100000:1.0:1767320576.965379:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983105024:0@lo:400:kworker.0 00000100:00100000:0.0:1767320576.965382:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104768 00000100:00100000:0.0:1767320576.965450:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1797 00000100:00100000:0.0:1767320576.965461:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983104768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320576.965500:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320576.965506:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983104768:12345-0@lo:400:kworker.0 Request processed in 48us (180us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320576.965513:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1797 00000100:00100000:0.0:1767320576.965527:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105024 00000100:00100000:0.0:1767320576.965530:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105024 00010000:00080000:1.0:1767320576.965532:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f963480 x1853163983104768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320592 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320576.965545:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983104768:0@lo:400:kworker.0 00000100:00100000:0.0:1767320576.965551:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1798 00000100:00100000:0.0:1767320576.965555:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983105024:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320576.965575:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320576.965586:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320576.965587:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983105024:12345-0@lo:400:kworker.0 Request processed in 31us (87us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320576.965593:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1798 00000100:00100000:0.0:1767320576.965602:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983104896 00000100:00100000:0.0:1767320576.965605:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983104896 00000100:00100000:0.0:1767320576.965617:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2377 00000100:00100000:0.0:1767320576.965621:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983104896:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320576.965644:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:1.0:1767320576.965657:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163983105024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320592 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320576.965676:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983104896:12345-0@lo:38:kworker.0 Request processed in 52us (89us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320576.965682:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2377 00000100:00080000:0.0:1767320576.965709:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320576.965713:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320576.965716:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983104896:0@lo:38:kworker.0 00000100:00100000:2.0:1767320576.965807:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960a80 x1853163983105152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320576.965824:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983105024:0@lo:400:kworker.0 00000100:00080000:2.0:1767320576.965824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320576.965827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320576.965829:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320576.965832:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320576.965834:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320576.965838:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320576.965855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163983105280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320576.965894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320576.965897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320576.965899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320576.965902:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5820 00000100:00080000:2.0:1767320576.965905:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320576.965907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320576.965917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963480 x1853163983105664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320576.965941:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983105536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320576.965971:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320576.965995:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320576.966000:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983105408:0@lo:400:kworker.0 00000100:00100000:2.0:1767320576.966021:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320576.966045:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105536 00000100:00100000:1.0:1767320576.966047:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105408 00000100:00100000:2.0:1767320576.966049:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105536 00000100:00100000:1.0:1767320576.966050:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105408 00000100:00100000:1.0:1767320576.966078:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1111 00000100:00100000:2.0:1767320576.966083:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112 00000100:00100000:1.0:1767320576.966084:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983105408:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320576.966088:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983105536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320576.966835:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983105536:12345-0@lo:400:kworker.0 Request processed in 742us (859us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320576.966844:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112 00000100:00100000:1.0:1767320576.967424:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983105408:12345-0@lo:400:kworker.0 Request processed in 1338us (1399us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320576.967433:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1111 00000100:00100000:0.0:1767320576.967646:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983105152:0@lo:38:kworker.0 00000100:00100000:0.0:1767320576.967695:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320576.967739:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983105280:0@lo:38:kworker.0 00000100:00100000:0.0:1767320576.967764:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320576.967805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983105664:0@lo:38:kworker.0 00000100:00100000:0.0:1767320576.967828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320576.967862:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105152 00000100:00100000:0.0:1767320576.967871:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105152 00000100:00100000:0.0:1767320576.967896:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105664 00000100:00100000:0.0:1767320576.967899:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105664 00000100:00100000:0.0:1767320576.967916:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2378 00000100:00100000:0.0:1767320576.967921:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983105152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320576.967939:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320576.967976:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983105152:12345-0@lo:38:kworker.0 Request processed in 55us (281us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320576.967982:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2378 00000100:00100000:0.0:1767320576.967993:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2379 00000100:00100000:0.0:1767320576.967997:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983105664:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320576.968005:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320576.968025:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983105664:12345-0@lo:38:kworker.0 Request processed in 26us (197us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320576.968043:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2379 00000100:00080000:0.0:1767320576.968090:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320576.968095:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320576.968098:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983105152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320576.968328:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105280 00000100:00100000:1.0:1767320576.968333:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105280 00000100:00100000:1.0:1767320576.968358:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 619 00000100:00100000:1.0:1767320576.968363:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983105280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320576.968373:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320576.968402:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983105280:12345-0@lo:38:kworker.0 Request processed in 38us (637us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320576.968408:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 619 00000100:00080000:0.0:1767320576.968527:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320576.968532:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320576.968535:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983105280:0@lo:38:kworker.0 00000100:00080000:0.0:1767320576.968544:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320576.968547:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320576.968550:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983105664:0@lo:38:kworker.0 00000100:00100000:3.0:1767320576.968870:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983105408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320576.968884:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983105536:0@lo:400:kworker.0 00010000:00010000:2.0:1767320577.156499:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320578.180637:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320579.204524:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320579.980256:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320579.982387:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689472 00000100:00100000:2.0:1767320579.982401:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689472 00000100:00100000:3.0:1767320579.982466:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:1.0:1767320579.982581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320579.987819:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689344 00000100:00100000:2.0:1767320579.987836:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689344 00000100:00100000:2.0:1767320579.987906:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1799 00000100:00100000:1.0:1767320579.987919:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2380 00000100:00100000:2.0:1767320579.987927:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915689472:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320579.987934:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915689344:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320579.987956:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320579.988051:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915689344:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 120us (5589us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320579.988058:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2380 00000100:00100000:2.0:1767320579.988123:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915689472:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 207us (7876us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320579.988130:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1799 00010000:00010000:1.0:1767320580.229016:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320580.551297:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320580.551329:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983105792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320580.551438:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320580.551954:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320580.551967:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983105920:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320580.552001:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320580.552334:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105792 00000100:00100000:1.0:1767320580.552343:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105792 00000100:00100000:1.0:1767320580.552419:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983105920 00000100:00100000:1.0:1767320580.552423:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983105920 00000100:00100000:1.0:1767320580.552442:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1204 00000100:00100000:1.0:1767320580.552451:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983105792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320580.552505:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320580.552556:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983105792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (1117us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320580.552563:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1204 00000100:00100000:1.0:1767320580.552580:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205 00000100:00100000:1.0:1767320580.552587:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983105920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320580.552602:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320580.552631:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983105920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 45us (629us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320580.552637:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205 00000100:00100000:3.0:1767320580.553147:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983105792:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320580.554214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320580.554228:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320580.554265:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983105920:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320581.252723:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320582.084405:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.084417:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320582.084486:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.084491:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.084495:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.084504:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.084509:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.084516:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320582.084790:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320582.084804:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983106048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.084874:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320582.084945:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106048 00000100:00100000:0.0:1767320582.084955:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106048 00000100:00100000:2.0:1767320582.084973:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961f80 x1853163983106176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320582.085007:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.085010:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.085013:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.085017:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.085020:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.085025:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320582.085038:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800 00000100:00100000:2.0:1767320582.085039:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963480 x1853163983106432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320582.085051:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1767320582.085053:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983106048:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320582.085057:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.085061:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.085066:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.085069:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.085073:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320582.085087:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163983106560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320582.085114:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983106048:12345-0@lo:400:kworker.0 Request processed in 65us (238us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320582.085125:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800 00000100:00080000:2.0:1767320582.085125:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320582.085132:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320582.085134:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320582.085139:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5825 00000100:00080000:2.0:1767320582.085142:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320582.085146:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320582.085158:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246680 x1853163983106944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320582.085178:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983106176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320582.085185:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983106688:0@lo:400:kworker.0 00000100:00100000:0.0:1767320582.085213:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.085230:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320582.085244:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983106432:0@lo:38:kworker.0 00000100:00100000:2.0:1767320582.085260:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320582.085301:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.085302:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983106816:0@lo:400:kworker.0 00000100:00100000:0.0:1767320582.085324:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983106560:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.085394:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320582.085403:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983106304:0@lo:400:kworker.0 00000100:00080000:3.0:1767320582.085455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320582.085486:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962a00 x1853163983106048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320598 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320582.085499:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983106048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320582.089327:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320582.089401:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983106944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320582.089448:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320582.089511:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106176 00000100:00100000:0.0:1767320582.089518:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106176 00000100:00100000:0.0:1767320582.089557:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106432 00000100:00100000:0.0:1767320582.089560:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106432 00000100:00100000:0.0:1767320582.089576:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106944 00000100:00100000:0.0:1767320582.089578:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106944 00000100:00100000:0.0:1767320582.089593:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2381 00000100:00100000:0.0:1767320582.089599:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983106176:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320582.089630:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320582.089684:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983106176:12345-0@lo:38:kworker.0 Request processed in 84us (4471us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320582.089691:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2381 00000100:00100000:0.0:1767320582.089707:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2382 00000100:00100000:0.0:1767320582.089711:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983106432:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320582.089721:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320582.089745:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983106432:12345-0@lo:38:kworker.0 Request processed in 32us (4473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320582.089751:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2382 00000100:00100000:3.0:1767320582.089891:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106560 00000100:00100000:3.0:1767320582.089896:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106560 00000100:00100000:3.0:1767320582.089926:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 620 00000100:00100000:3.0:1767320582.089932:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983106560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320582.089942:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320582.089972:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983106560:12345-0@lo:38:kworker.0 Request processed in 39us (654us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320582.089978:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 620 00000100:00100000:3.0:1767320582.089999:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2383 00000100:00100000:3.0:1767320582.090003:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983106944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320582.090012:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320582.090037:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983106944:12345-0@lo:38:kworker.0 Request processed in 33us (588us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320582.090043:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2383 00000100:00100000:2.0:1767320582.090055:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320582.090093:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106688 00000100:00100000:2.0:1767320582.090098:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106688 00000100:00080000:3.0:1767320582.090129:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320582.090146:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106816 00000100:00100000:2.0:1767320582.090151:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106816 00000100:00080000:3.0:1767320582.090179:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320582.090191:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113 00000100:00100000:3.0:1767320582.090205:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1114 00000100:00100000:2.0:1767320582.090207:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983106688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.090218:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983106816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.090311:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983106688:12345-0@lo:400:kworker.0 Request processed in 112us (5081us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320582.090319:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113 00000100:00080000:0.0:1767320582.090422:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320582.090428:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320582.090431:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983106176:0@lo:38:kworker.0 00000100:00080000:0.0:1767320582.090440:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320582.090444:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320582.090447:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983106432:0@lo:38:kworker.0 00000100:00080000:0.0:1767320582.090456:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320582.090459:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320582.090462:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983106560:0@lo:38:kworker.0 00000100:00080000:0.0:1767320582.090469:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320582.090472:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320582.090475:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983106944:0@lo:38:kworker.0 00000100:00100000:1.0:1767320582.091346:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320582.091659:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983106304 00000100:00100000:0.0:1767320582.091666:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983106304 00000100:00100000:0.0:1767320582.091726:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1801 00000100:00100000:0.0:1767320582.091734:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983106304:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320582.091939:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983106304:12345-0@lo:400:kworker.0 Request processed in 205us (594us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320582.091946:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1801 00000100:00080000:1.0:1767320582.092170:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320582.092202:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163983106304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320598 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320582.092217:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983106304:0@lo:400:kworker.0 00000100:00100000:3.0:1767320582.092320:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983106816:12345-0@lo:400:kworker.0 Request processed in 2104us (2267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320582.092329:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1114 00000100:00100000:2.0:1767320582.093532:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983106816:0@lo:400:kworker.0 00000100:00100000:2.0:1767320582.094159:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983106688:0@lo:400:kworker.0 00010000:00010000:1.0:1767320582.277116:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320583.300667:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320584.325935:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320585.101719:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320585.102408:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689600 00000100:00100000:0.0:1767320585.102420:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689600 00000100:00100000:0.0:1767320585.102475:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2384 00000100:00100000:0.0:1767320585.102484:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915689600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320585.102507:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320585.102573:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915689600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 89us (864us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320585.102589:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2384 00000100:00100000:3.0:1767320585.103686:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320585.103777:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689728 00000100:00100000:1.0:1767320585.103794:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689728 00000100:00100000:1.0:1767320585.103867:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1802 00000100:00100000:1.0:1767320585.103876:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915689728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320585.103942:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915689728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 67us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320585.103950:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1802 00000100:00080000:3.0:1767320585.104382:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320585.348620:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320585.668606:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983107072:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320585.668606:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983107200:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320585.668693:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320585.668850:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320585.668904:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107072 00000100:00100000:0.0:1767320585.668914:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107072 00000100:00100000:1.0:1767320585.668946:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107200 00000100:00100000:1.0:1767320585.668952:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107200 00000100:00080000:3.0:1767320585.669019:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320585.669024:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320585.669032:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1206 00000100:00100000:0.0:1767320585.669043:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983107072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320585.669075:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320585.669133:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983107072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (441us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320585.669141:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1206 00000100:00100000:0.0:1767320585.669158:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1207 00000100:00100000:0.0:1767320585.669165:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983107200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320585.669181:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320585.669219:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983107200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (360us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320585.669225:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1207 00000100:00100000:3.0:1767320585.669309:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983107072:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320585.669343:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983107200:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320586.372689:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320587.204413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320587.204424:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320587.204489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320587.204494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320587.204497:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320587.204505:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5830 00000100:00080000:0.0:1767320587.204509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320587.204515:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320587.204534:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198bb80 x1853163983107456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320587.204586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320587.204589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320587.204591:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320587.204595:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5830 00000100:00080000:0.0:1767320587.204597:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320587.204602:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320587.204612:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246680 x1853163983107712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320587.204618:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983107456:0@lo:38:kworker.0 00000100:00080000:0.0:1767320587.204622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320587.204624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320587.204626:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320587.204629:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5830 00000100:00080000:0.0:1767320587.204632:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320587.204635:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320587.204644:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624000 x1853163983107840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320587.204673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320587.204676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320587.204677:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320587.204682:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5830 00000100:00100000:1.0:1767320587.204683:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320587.204684:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320587.204687:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320587.204700:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625880 x1853163983108224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320587.204722:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983107968:0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.204753:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.204811:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320587.204817:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983108096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320587.204837:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.204868:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107968 00000100:00100000:0.0:1767320587.204876:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107968 00000100:00100000:0.0:1767320587.204920:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108096 00000100:00100000:0.0:1767320587.204923:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108096 00000100:00100000:0.0:1767320587.204945:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115 00000100:00100000:0.0:1767320587.204953:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983107968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320587.204954:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983107712:0@lo:38:kworker.0 00000100:00100000:1.0:1767320587.204974:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320587.204987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983107840:0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.204997:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983107968:12345-0@lo:400:kworker.0 Request processed in 45us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320587.205004:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115 00000100:00100000:1.0:1767320587.205007:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.205018:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116 00000100:00100000:0.0:1767320587.205024:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983108096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320587.205026:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983108224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320587.205045:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320587.205047:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983108096:12345-0@lo:400:kworker.0 Request processed in 24us (210us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320587.205053:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116 00000100:00100000:1.0:1767320587.205069:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107456 00000100:00100000:1.0:1767320587.205071:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107456 00000100:00100000:0.0:1767320587.205083:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983108096:0@lo:400:kworker.0 00000100:00100000:1.0:1767320587.205090:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107712 00000100:00100000:0.0:1767320587.205092:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983107968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320587.205092:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107712 00000100:00100000:1.0:1767320587.205103:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108224 00000100:00100000:1.0:1767320587.205105:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108224 00000100:00100000:1.0:1767320587.205114:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2385 00000100:00100000:1.0:1767320587.205118:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983107456:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.205123:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107840 00000100:00100000:0.0:1767320587.205125:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107840 00010000:00020000:1.0:1767320587.205136:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320587.205140:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 621 00000100:00100000:0.0:1767320587.205144:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983107840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320587.205152:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320587.205168:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983107456:12345-0@lo:38:kworker.0 Request processed in 47us (484us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.205171:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983107840:12345-0@lo:38:kworker.0 Request processed in 26us (163us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.205177:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 621 00000100:00100000:1.0:1767320587.205177:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2385 00000100:00100000:1.0:1767320587.205188:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2386 00000100:00100000:1.0:1767320587.205192:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983107712:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320587.205193:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2387 00000100:00100000:0.0:1767320587.205197:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983108224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320587.205201:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767320587.205204:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320587.205221:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983108224:12345-0@lo:38:kworker.0 Request processed in 22us (175us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320587.205221:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983107712:12345-0@lo:38:kworker.0 Request processed in 26us (245us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320587.205226:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2387 00000100:00100000:1.0:1767320587.205239:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2386 00000100:00080000:1.0:1767320587.205268:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320587.205290:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983107328:0@lo:400:kworker.0 00000100:00080000:1.0:1767320587.205292:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320587.205295:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983107712:0@lo:38:kworker.0 00000100:00080000:1.0:1767320587.205305:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320587.205307:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320587.205310:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983107840:0@lo:38:kworker.0 00000100:00080000:1.0:1767320587.205317:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320587.205318:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320587.205319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320587.205322:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983108224:0@lo:38:kworker.0 00000100:00080000:1.0:1767320587.205329:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320587.205332:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320587.205334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983107456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320587.205337:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320587.205341:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983107584:0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.205361:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320587.205390:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107328 00000100:00100000:2.0:1767320587.205393:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107328 00000100:00100000:2.0:1767320587.205418:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983107584 00000100:00100000:2.0:1767320587.205423:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983107584 00000100:00100000:2.0:1767320587.205438:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1803 00000100:00100000:2.0:1767320587.205444:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983107328:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.205476:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983107328:12345-0@lo:400:kworker.0 Request processed in 31us (156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320587.205482:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1803 00000100:00100000:2.0:1767320587.205493:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804 00000100:00100000:2.0:1767320587.205499:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983107584:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320587.205522:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983107584:12345-0@lo:400:kworker.0 Request processed in 24us (161us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320587.205528:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804 00010000:00080000:2.0:1767320587.205562:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e245880 x1853163983107584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320603 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320587.205572:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983107584:0@lo:400:kworker.0 00010000:00080000:2.0:1767320587.205580:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631989500 x1853163983107328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320603 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320587.205588:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983107328:0@lo:400:kworker.0 00000100:00080000:3.0:1767320587.205722:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320587.205726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320587.205728:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320587.396586:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320588.423476:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767320589.200945:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915689856 offset 0 length 520: 4 00000100:00100000:3.0:1767320589.201027:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320589.201156:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690112 00000100:00100000:0.0:1767320589.201170:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690112 00000100:00100000:3.0:1767320589.201289:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320589.201349:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915689984 00000100:00100000:0.0:1767320589.201355:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915689984 00000100:00100000:0.0:1767320589.201387:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1805 00000100:00100000:0.0:1767320589.201400:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915690112:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767320589.201605:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320589.201720:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2388 00000100:00100000:3.0:1767320589.201726:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915689984:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320589.201751:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320589.201829:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915690112:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 430us (800us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320589.201838:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1805 00000100:00100000:3.0:1767320589.201912:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915689984:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 183us (641us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320589.201919:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2388 00010000:00010000:1.0:1767320589.444835:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320590.468766:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320590.789209:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983108352:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320590.789367:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320590.789462:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320590.789470:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983108480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320590.789508:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320590.790306:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108352 00000100:00100000:1.0:1767320590.790319:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108352 00000100:00100000:1.0:1767320590.790411:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108480 00000100:00100000:1.0:1767320590.790415:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108480 00000100:00100000:1.0:1767320590.790450:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208 00000100:00100000:1.0:1767320590.790461:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983108352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320590.790512:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320590.790588:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983108352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (1220us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320590.790597:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208 00000100:00100000:1.0:1767320590.790621:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209 00000100:00100000:1.0:1767320590.790630:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983108480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320590.790650:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320590.790684:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983108480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (1176us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320590.790690:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209 00000100:00080000:2.0:1767320590.791315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320590.791322:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320590.791381:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631989500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983108480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320590.791401:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983108352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320591.300530:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320566/real 1767320566] req@ffff934604583480 x1853163983102080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320591 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320591.300578:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604583480 x1853163983102080/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320591 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320591.300604:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320591.300650:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320591.300655:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983102080:0@lo:250:kworker.0 00000100:00080000:3.0:1767320591.300844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320591.300852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320591.300856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320591.300861:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5815 00000100:00080000:3.0:1767320591.300865:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320591.300873:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320591.300917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2300 x1853163983108608/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320591.300985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320591.300989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320591.300991:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320591.300995:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5835 00000100:00080000:3.0:1767320591.300998:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320591.301002:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320591.301019:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2d80 x1853163983108864/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320591.301036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320591.301039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320591.301041:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320591.301045:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5835 00000100:00080000:3.0:1767320591.301047:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320591.301051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320591.301080:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0000 x1853163983109120/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320591.301090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320591.301093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320591.301095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320591.301099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5835 00000100:00080000:3.0:1767320591.301101:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320591.301105:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320591.301116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b2680 x1853163983109248/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320591.301126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320591.301130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320591.301131:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320591.301135:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5835 00000100:00080000:3.0:1767320591.301137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320591.301141:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320591.301302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0380 x1853163983109376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320591.301395:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983108992:0@lo:400:kworker.0 00000100:00100000:2.0:1767320591.301494:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320591.301578:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320591.301585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983108736:0@lo:400:kworker.0 00000100:00100000:2.0:1767320591.301610:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320591.301668:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108992 00000100:00100000:2.0:1767320591.301677:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108992 00000100:00100000:2.0:1767320591.301739:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108736 00000100:00100000:2.0:1767320591.301743:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108736 00000100:00100000:2.0:1767320591.301773:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1806 00000100:00100000:2.0:1767320591.301783:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983108992:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320591.301844:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983108992:12345-0@lo:400:kworker.0 Request processed in 60us (349us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320591.301852:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1806 00000100:00100000:2.0:1767320591.301869:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1807 00000100:00100000:2.0:1767320591.301875:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983108736:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320591.301901:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983108736:12345-0@lo:400:kworker.0 Request processed in 27us (290us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320591.301907:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1807 00000100:00080000:2.0:1767320591.301925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320591.301958:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b3800 x1853163983108736/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320591.301972:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983108736:0@lo:400:kworker.0 00010000:00080000:2.0:1767320591.301982:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b1180 x1853163983108992/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320591.301990:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983108992:0@lo:400:kworker.0 00000100:00100000:1.0:1767320591.302523:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983108608:0@lo:250:kworker.0 00000400:00000100:1.0:1767320591.302559:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983108608 offset 0 length 520: 4 00000100:00100000:1.0:1767320591.302586:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983108864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320591.302608:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320591.302641:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983109120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320591.302662:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320591.302684:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983109248:0@lo:38:kworker.0 00000100:00100000:1.0:1767320591.302707:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320591.302724:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983109376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320591.302746:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320591.302779:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983108864 00000100:00100000:1.0:1767320591.302784:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983108864 00000100:00100000:1.0:1767320591.302805:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109120 00000100:00100000:1.0:1767320591.302807:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109120 00000100:00100000:1.0:1767320591.302817:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109376 00000100:00100000:1.0:1767320591.302819:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109376 00000100:00100000:1.0:1767320591.302827:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2389 00000100:00100000:1.0:1767320591.302833:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983108864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320591.302855:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320591.303951:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2390 00000100:00100000:0.0:1767320591.303963:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983109120:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320591.303985:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320591.304092:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983109120:12345-0@lo:38:kworker.0 Request processed in 129us (1427us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320591.304101:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2390 00000100:00100000:0.0:1767320591.304128:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2391 00000100:00100000:0.0:1767320591.304132:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983109376:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320591.304141:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320591.304183:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983109376:12345-0@lo:38:kworker.0 Request processed in 50us (1436us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320591.304190:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2391 00000100:00080000:0.0:1767320591.304260:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320591.304266:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320591.304304:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983109120:0@lo:38:kworker.0 00000100:00080000:0.0:1767320591.304318:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320591.304322:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320591.304325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983109376:0@lo:38:kworker.0 00000100:00100000:3.0:1767320591.304792:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109248 00000100:00100000:3.0:1767320591.304801:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109248 00000100:00100000:3.0:1767320591.304846:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 622 00000100:00100000:3.0:1767320591.304853:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983109248:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320591.304868:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320591.304914:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983109248:12345-0@lo:38:kworker.0 Request processed in 60us (2205us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320591.304921:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 622 00000100:00080000:0.0:1767320591.306261:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320591.306302:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320591.306307:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983109248:0@lo:38:kworker.0 00000100:00100000:1.0:1767320591.332753:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983108864:12345-0@lo:38:kworker.0 Request processed in 29911us (30135us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320591.332767:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2389 00000100:00080000:2.0:1767320591.337345:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320591.337363:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320591.337370:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983108864:0@lo:38:kworker.0 00010000:00010000:1.0:1767320591.493214:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320592.516553:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320593.540609:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320594.324930:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320594.325102:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320594.326287:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690368 00000100:00100000:2.0:1767320594.326306:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690368 00000100:00100000:3.0:1767320594.326524:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690240 00000100:00100000:3.0:1767320594.326530:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690240 00000100:00100000:1.0:1767320594.326829:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1808 00000100:00100000:1.0:1767320594.326844:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915690368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320594.326962:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915690368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 123us (2048us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320594.326970:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1808 00000100:00100000:1.0:1767320594.326991:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2392 00000100:00100000:1.0:1767320594.326995:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915690240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320594.327022:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320594.327049:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915690240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (1948us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320594.327057:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2392 00000100:00080000:0.0:1767320594.329331:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320594.565387:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320595.588474:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320595.909464:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983109504:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320595.909986:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320595.910594:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109504 00000100:00100000:1.0:1767320595.910609:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109504 00000100:00100000:0.0:1767320595.910966:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983109632:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320595.911036:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320595.911084:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109632 00000100:00100000:0.0:1767320595.911089:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109632 00000100:00100000:1.0:1767320595.911232:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1210 00000100:00100000:1.0:1767320595.911247:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983109504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320595.911320:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320595.911671:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983109504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 427us (1686us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320595.912019:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320595.912025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320595.912084:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983109504:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320595.916325:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211 00000100:00100000:0.0:1767320595.916371:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983109632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320595.916440:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320595.916943:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983109632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 599us (5905us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320595.916954:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211 00000100:00100000:2.0:1767320595.917115:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983109632:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320595.922262:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1210 00000100:00080000:2.0:1767320596.420654:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320596.420670:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320596.420746:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320596.420751:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320596.420755:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320596.420767:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5839 00000100:00080000:2.0:1767320596.420774:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320596.420784:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320596.420814:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983109888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320596.420874:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983109760:0@lo:400:kworker.0 00000100:00100000:0.0:1767320596.420975:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320596.432850:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320596.432858:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320596.432862:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320596.432872:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5839 00000100:00080000:2.0:1767320596.432877:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320596.432886:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320596.432918:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2d80 x1853163983110144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320596.432933:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320596.432936:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320596.432938:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320596.432942:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5839 00000100:00080000:2.0:1767320596.432945:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320596.432948:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320596.432961:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2a00 x1853163983110272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320596.432971:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320596.432975:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320596.432976:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320596.432980:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5839 00000100:00080000:2.0:1767320596.432982:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320596.432986:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320596.432996:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c0e00 x1853163983110400/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320596.433051:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983109888:0@lo:38:kworker.0 00000100:00100000:1.0:1767320596.433740:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983110016:0@lo:400:kworker.0 00000100:00100000:0.0:1767320596.435368:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320596.435743:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109760 00000100:00100000:1.0:1767320596.435755:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109760 00000100:00100000:1.0:1767320596.435839:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983109888 00000100:00100000:1.0:1767320596.435843:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983109888 00000100:00100000:1.0:1767320596.435864:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1809 00000100:00100000:1.0:1767320596.435877:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983109760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320596.435994:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983109760:12345-0@lo:400:kworker.0 Request processed in 120us (15019us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320596.436004:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1809 00000100:00100000:1.0:1767320596.436026:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2393 00000100:00100000:1.0:1767320596.436030:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983109888:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320596.436041:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767320596.436064:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320596.436081:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110016 00000100:00100000:3.0:1767320596.436086:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110016 00000100:00100000:1.0:1767320596.436101:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983109888:12345-0@lo:38:kworker.0 Request processed in 70us (742us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320596.436109:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2393 00000100:00100000:3.0:1767320596.436117:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1810 00000100:00100000:3.0:1767320596.436124:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983110016:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320596.436157:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320596.436161:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983110016:12345-0@lo:400:kworker.0 Request processed in 39us (121us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320596.436163:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320596.436167:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983109888:0@lo:38:kworker.0 00000100:00100000:3.0:1767320596.436168:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1810 00000100:00100000:1.0:1767320596.436179:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983110144:0@lo:38:kworker.0 00010000:00080000:3.0:1767320596.436201:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1180 x1853163983110016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320612 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320596.436214:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983110016:0@lo:400:kworker.0 00000100:00100000:1.0:1767320596.436216:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320596.436232:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983110272:0@lo:38:kworker.0 00000100:00100000:3.0:1767320596.436237:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110144 00000100:00100000:3.0:1767320596.436240:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110144 00000100:00100000:3.0:1767320596.436256:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2394 00000100:00100000:1.0:1767320596.436257:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320596.436261:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983110144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320596.436293:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320596.436310:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983110400:0@lo:38:kworker.0 00000100:00100000:3.0:1767320596.436319:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983110144:12345-0@lo:38:kworker.0 Request processed in 57us (102us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320596.436325:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2394 00000100:00100000:1.0:1767320596.436331:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320596.436336:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110400 00000100:00100000:3.0:1767320596.436339:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110400 00000100:00100000:3.0:1767320596.436355:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2395 00000100:00080000:1.0:1767320596.436358:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320596.436358:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983110400:12345-0@lo:38:kworker.0 00000100:00080000:1.0:1767320596.436362:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320596.436365:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983110144:0@lo:38:kworker.0 00010000:00020000:3.0:1767320596.436367:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320596.436408:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983110400:12345-0@lo:38:kworker.0 Request processed in 47us (75us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320596.436414:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2395 00000100:00080000:3.0:1767320596.436438:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320596.436443:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320596.436446:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983110400:0@lo:38:kworker.0 00000100:00080000:0.0:1767320596.438221:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320596.438263:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110272 00000100:00100000:0.0:1767320596.438323:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110272 00000100:00100000:0.0:1767320596.438399:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 623 00000100:00100000:0.0:1767320596.438408:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983110272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320596.438435:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320596.438528:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983110272:12345-0@lo:38:kworker.0 Request processed in 120us (2268us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320596.438536:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 623 00010000:00080000:0.0:1767320596.438589:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2680 x1853163983109760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320612 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320596.438606:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983109760:0@lo:400:kworker.0 00000100:00080000:1.0:1767320596.444027:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320596.444050:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320596.444056:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983110272:0@lo:38:kworker.0 00010000:00010000:2.0:1767320596.614341:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320597.636714:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320598.661436:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320599.438133:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320599.438204:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320599.438503:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690496 00000100:00100000:2.0:1767320599.438522:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690496 00000100:00100000:2.0:1767320599.438592:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690624 00000100:00100000:2.0:1767320599.438596:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690624 00000100:00080000:0.0:1767320599.438872:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320599.438971:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1811 00000100:00100000:2.0:1767320599.438994:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915690624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320599.439119:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915690624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 128us (912us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320599.439129:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1811 00000100:00100000:2.0:1767320599.439163:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2396 00000100:00100000:2.0:1767320599.439169:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915690496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320599.439196:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320599.439228:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915690496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (1105us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320599.439234:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2396 00010000:00010000:2.0:1767320599.685592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320600.709193:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320601.028643:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983110528:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320601.028690:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983110656:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320601.028754:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320601.028812:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320601.028965:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110528 00000100:00100000:1.0:1767320601.028974:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110528 00000100:00100000:1.0:1767320601.029034:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110656 00000100:00100000:1.0:1767320601.029037:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110656 00000100:00100000:1.0:1767320601.029072:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212 00000100:00100000:1.0:1767320601.029081:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983110528:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320601.029111:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320601.029568:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983110528:12345-0@lo:13:osp-pre-0-1.0 Request processed in 485us (829us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320601.029576:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212 00000100:00100000:1.0:1767320601.029594:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1213 00000100:00100000:1.0:1767320601.029601:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983110656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320601.029616:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320601.029645:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983110656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (836us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320601.029650:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1213 00000100:00080000:1.0:1767320601.029668:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320601.029673:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320601.029788:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983110656:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320601.035735:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983110528:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320601.545309:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320601.545321:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320601.545407:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320601.545412:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320601.545416:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320601.545424:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5844 00000100:00080000:0.0:1767320601.545438:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320601.545449:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320601.545476:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51180 x1853163983110912/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320601.545495:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:0.0:1767320601.545513:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320601.545518:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320601.545520:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320601.545524:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5844 00000100:00080000:0.0:1767320601.545532:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320601.545532:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983111040:0@lo:400:kworker.0 00000100:00080000:0.0:1767320601.545543:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320601.545559:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f53100 x1853163983111168/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320601.545569:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320601.545573:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320601.545576:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320601.545580:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5844 00000100:00080000:0.0:1767320601.545583:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320601.545587:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320601.545601:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f50000 x1853163983111296/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320601.545644:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320601.545646:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320601.545651:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320601.545653:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320601.545657:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5844 00000100:00080000:0.0:1767320601.545660:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320601.545665:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320601.545676:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51880 x1853163983111680/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320601.545704:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983110784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320601.545714:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983111552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320601.545737:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320601.545799:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320601.545812:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983111424:0@lo:400:kworker.0 00000100:00100000:3.0:1767320601.545858:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320601.545879:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111040 00000100:00100000:0.0:1767320601.545887:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111040 00000100:00100000:3.0:1767320601.545925:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111552 00000100:00100000:3.0:1767320601.545934:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111552 00000100:00100000:0.0:1767320601.545946:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110784 00000100:00100000:0.0:1767320601.545953:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110784 00000100:00100000:3.0:1767320601.545986:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111424 00000100:00100000:3.0:1767320601.545992:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111424 00000100:00100000:0.0:1767320601.545994:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812 00000100:00100000:0.0:1767320601.546010:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983111040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320601.546033:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1117 00000100:00100000:3.0:1767320601.546045:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983111552:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320601.546084:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983111040:12345-0@lo:400:kworker.0 Request processed in 78us (442us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320601.546095:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812 00000100:00100000:3.0:1767320601.546118:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983111552:12345-0@lo:400:kworker.0 Request processed in 74us (318us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320601.546126:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1117 00000100:00100000:0.0:1767320601.546135:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1813 00000100:00100000:0.0:1767320601.546149:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983110784:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320601.546152:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1767320601.546170:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983111424:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320601.546215:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983110784:12345-0@lo:400:kworker.0 Request processed in 68us (474us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320601.546225:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1813 00000100:00100000:3.0:1767320601.546233:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983111424:12345-0@lo:400:kworker.0 Request processed in 63us (371us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320601.546244:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1767320601.546346:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983111424:0@lo:400:kworker.0 00000100:00080000:1.0:1767320601.546453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320601.546470:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983110912:0@lo:38:kworker.0 00000100:00100000:0.0:1767320601.546923:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f52300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983111552:0@lo:400:kworker.0 00000100:00100000:1.0:1767320601.552653:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320601.552724:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983111168:0@lo:38:kworker.0 00000100:00100000:2.0:1767320601.553317:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983110912 00000100:00100000:2.0:1767320601.553327:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983110912 00000100:00100000:2.0:1767320601.553397:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2397 00000100:00100000:2.0:1767320601.553406:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983110912:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320601.553580:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320601.553636:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983110912:12345-0@lo:38:kworker.0 Request processed in 229us (989us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320601.553643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2397 00000100:00100000:1.0:1767320601.553894:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320601.553921:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983111296:0@lo:38:kworker.0 00000100:00100000:1.0:1767320601.553946:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320601.553971:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983111680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320601.553991:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320601.554032:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320601.554037:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320601.554041:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983110912:0@lo:38:kworker.0 00000100:00100000:1.0:1767320601.554067:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111296 00000100:00100000:1.0:1767320601.554072:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111296 00000100:00100000:1.0:1767320601.554095:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 624 00000100:00100000:1.0:1767320601.554099:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983111296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320601.554110:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320601.554365:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111168 00000100:00100000:0.0:1767320601.554370:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111168 00000100:00100000:0.0:1767320601.554390:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111680 00000100:00100000:0.0:1767320601.554393:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111680 00000100:00100000:0.0:1767320601.554408:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2398 00000100:00100000:0.0:1767320601.554413:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983111168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320601.554424:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320601.554666:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2399 00000100:00100000:2.0:1767320601.554672:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983111680:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320601.554682:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:3.0:1767320601.557261:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934610f51c00 x1853163983111040/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320617 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320601.557312:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983111040:0@lo:400:kworker.0 00010000:00080000:3.0:1767320601.557334:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626d80 x1853163983110784/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320617 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320601.557344:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983110784:0@lo:400:kworker.0 00000100:00100000:0.0:1767320601.565165:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983111168:12345-0@lo:38:kworker.0 Request processed in 10736us (11255us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320601.565184:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2398 00000100:00080000:3.0:1767320601.565248:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320601.565262:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320601.565268:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983111168:0@lo:38:kworker.0 00000100:00100000:1.0:1767320601.584067:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983111296:12345-0@lo:38:kworker.0 Request processed in 29955us (30109us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320601.584085:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 624 00000100:00080000:3.0:1767320601.584251:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320601.584264:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320601.584336:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f50000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983111296:0@lo:38:kworker.0 00000100:00100000:2.0:1767320601.584419:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983111680:12345-0@lo:38:kworker.0 Request processed in 29743us (30423us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320601.584429:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2399 00000100:00080000:3.0:1767320601.584597:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320601.584603:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320601.584607:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983111680:0@lo:38:kworker.0 00010000:00010000:1.0:1767320601.734883:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320602.756600:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320603.788898:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320604.556089:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320604.557193:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320604.559567:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690880 00000100:00100000:0.0:1767320604.559587:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690880 00000100:00100000:0.0:1767320604.559674:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915690752 00000100:00100000:0.0:1767320604.559688:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915690752 00000100:00100000:0.0:1767320604.559714:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1814 00000100:00100000:0.0:1767320604.559724:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915690880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320604.559881:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2400 00000100:00100000:3.0:1767320604.559888:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915690752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320604.559910:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320604.559977:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915690752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 88us (2788us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320604.559984:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2400 00000100:00100000:0.0:1767320604.560294:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915690880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 571us (4215us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320604.560301:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1814 00000100:00080000:1.0:1767320604.560417:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320604.804660:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320605.829257:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320606.148515:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983111808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320606.148585:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320606.148674:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111808 00000100:00100000:1.0:1767320606.148683:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111808 00000100:00100000:0.0:1767320606.148701:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983111936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320606.148737:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320606.148741:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214 00000100:00100000:1.0:1767320606.148753:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983111808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320606.148784:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320606.148794:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983111936 00000100:00100000:0.0:1767320606.148813:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983111936 00000100:00100000:1.0:1767320606.148835:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983111808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320606.148845:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214 00000100:00100000:3.0:1767320606.149154:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983111808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320606.157705:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215 00000100:00100000:0.0:1767320606.157742:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983111936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320606.157789:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320606.158099:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983111936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 378us (9360us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320606.158108:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215 00000100:00080000:0.0:1767320606.158145:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320606.158150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320606.158773:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983111936:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320606.660475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320606.660487:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320606.660587:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767320606.660731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320606.660736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320606.660740:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320606.660748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5850 00000100:00080000:3.0:1767320606.660752:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320606.660757:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320606.660781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1180 x1853163983112192/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320606.660866:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983112192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320606.660942:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320606.661054:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112192 00000100:00100000:0.0:1767320606.661062:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112192 00000100:00100000:0.0:1767320606.661103:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2401 00000100:00100000:0.0:1767320606.661120:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983112192:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320606.661140:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320606.661195:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983112192:12345-0@lo:38:kworker.0 Request processed in 64us (241us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320606.661202:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2401 00000100:00080000:1.0:1767320606.661417:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320606.661422:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320606.661425:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983112192:0@lo:38:kworker.0 00000100:00100000:2.0:1767320606.662716:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983112320:0@lo:400:kworker.0 00000100:00100000:2.0:1767320606.662773:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320606.662828:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983112064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320606.662850:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320606.663026:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112320 00000100:00100000:0.0:1767320606.663033:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112320 00000100:00100000:0.0:1767320606.663089:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112064 00000100:00100000:0.0:1767320606.663093:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112064 00000100:00100000:0.0:1767320606.663134:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1815 00000100:00100000:0.0:1767320606.663142:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983112320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320606.663213:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983112320:12345-0@lo:400:kworker.0 Request processed in 72us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320606.663221:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1815 00000100:00100000:0.0:1767320606.663237:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816 00000100:00100000:0.0:1767320606.663245:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983112064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320606.663294:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983112064:12345-0@lo:400:kworker.0 Request processed in 49us (443us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320606.663300:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816 00000100:00080000:2.0:1767320606.663735:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320606.663766:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b3800 x1853163983112320/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320622 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320606.663779:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983112320:0@lo:400:kworker.0 00010000:00080000:2.0:1767320606.663803:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b0000 x1853163983112064/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320622 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320606.663818:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983112064:0@lo:400:kworker.0 00000100:00080000:3.0:1767320606.667047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320606.667057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320606.667061:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320606.667068:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5850 00000100:00080000:3.0:1767320606.667073:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320606.667079:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320606.667130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0000 x1853163983112448/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320606.667156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320606.667159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320606.667161:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320606.667165:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5850 00000100:00080000:3.0:1767320606.667167:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320606.667171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320606.667193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b3800 x1853163983112576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320606.667235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320606.667238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320606.667240:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320606.667243:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5850 00000100:00080000:3.0:1767320606.667246:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320606.667249:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320606.667265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51880 x1853163983112960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320606.667312:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983112832:0@lo:400:kworker.0 00000100:00100000:1.0:1767320606.667313:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983112448:0@lo:38:kworker.0 00000100:00100000:2.0:1767320606.667401:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983112704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320606.670021:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320606.670171:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983112576:0@lo:38:kworker.0 00000100:00100000:1.0:1767320606.670221:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320606.670247:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983112960:0@lo:38:kworker.0 00000100:00100000:1.0:1767320606.670286:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320606.670402:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320606.670452:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112832 00000100:00100000:3.0:1767320606.670460:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112832 00000100:00100000:3.0:1767320606.670516:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767320606.670527:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983112832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320606.670580:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983112832:12345-0@lo:400:kworker.0 Request processed in 55us (177us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320606.670588:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119 00000100:00100000:3.0:1767320606.670631:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983112832:0@lo:400:kworker.0 00000100:00100000:3.0:1767320606.670678:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112576 00000100:00100000:3.0:1767320606.670682:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112576 00000100:00100000:3.0:1767320606.670702:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 625 00000100:00100000:3.0:1767320606.670707:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983112576:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320606.670730:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320606.670761:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983112576:12345-0@lo:38:kworker.0 Request processed in 52us (538us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320606.670767:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 625 00000100:00080000:1.0:1767320606.672884:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320606.672896:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320606.672901:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983112576:0@lo:38:kworker.0 00000100:00100000:2.0:1767320606.673082:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320606.674703:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112704 00000100:00100000:1.0:1767320606.674716:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112704 00000100:00100000:1.0:1767320606.674785:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120 00000100:00100000:1.0:1767320606.674796:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983112704:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320606.674879:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983112704:12345-0@lo:400:kworker.0 Request processed in 84us (1797us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320606.674887:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120 00000100:00100000:2.0:1767320606.675329:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983112704:0@lo:400:kworker.0 00000100:00100000:0.0:1767320606.676009:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112448 00000100:00100000:0.0:1767320606.676017:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112448 00000100:00100000:0.0:1767320606.676075:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983112960 00000100:00100000:0.0:1767320606.676078:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983112960 00000100:00100000:0.0:1767320606.676105:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2402 00000100:00100000:0.0:1767320606.676111:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983112448:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320606.676132:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2403 00000100:00100000:2.0:1767320606.676138:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983112960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320606.676206:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320606.676261:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983112448:12345-0@lo:38:kworker.0 Request processed in 149us (6247us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320606.676268:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2402 00000100:00080000:1.0:1767320606.676375:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320606.676381:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320606.676384:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983112448:0@lo:38:kworker.0 00010000:00020000:2.0:1767320606.680161:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320606.680294:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983112960:12345-0@lo:38:kworker.0 Request processed in 4152us (10020us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320606.680309:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2403 00000100:00080000:1.0:1767320606.680446:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320606.680451:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320606.680455:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983112960:0@lo:38:kworker.0 00010000:00010000:1.0:1767320606.852516:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320607.880845:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320608.901192:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320609.676003:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320609.676058:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320609.676456:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691136 00000100:00100000:2.0:1767320609.676465:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691136 00000100:00100000:2.0:1767320609.676543:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691008 00000100:00100000:2.0:1767320609.676547:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691008 00000100:00100000:2.0:1767320609.676566:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1817 00000100:00100000:2.0:1767320609.676576:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915691136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320609.676638:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2404 00000100:00100000:1.0:1767320609.676644:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915691008:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767320609.676657:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915691136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (666us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320609.676664:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1817 00010000:00020000:1.0:1767320609.676673:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320609.676700:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915691008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (643us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320609.676706:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2404 00000100:00080000:3.0:1767320609.676896:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320609.924506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320610.949562:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320611.268834:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983113088:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320611.268929:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320611.269201:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113088 00000100:00100000:0.0:1767320611.269209:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113088 00000100:00100000:2.0:1767320611.269513:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983113216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320611.269561:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.270503:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113216 00000100:00100000:1.0:1767320611.270511:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113216 00000100:00100000:0.0:1767320611.271799:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1216 00000100:00100000:0.0:1767320611.271814:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983113088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320611.271847:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320611.271913:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983113088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (2982us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320611.271921:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1216 00000100:00100000:1.0:1767320611.273262:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217 00000100:00100000:1.0:1767320611.273290:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983113216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320611.273326:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320611.273402:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983113216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 122us (3841us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320611.273410:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217 00000100:00100000:1.0:1767320611.273469:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983113088:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320611.278321:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983113216:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320611.285975:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320611.285986:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320611.780729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320611.780738:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320611.780791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320611.780795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320611.780798:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320611.780807:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5855 00000100:00080000:0.0:1767320611.780812:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320611.780818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320611.780838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358a80 x1853163983113472/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320611.780864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320611.780868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320611.780869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320611.780872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5855 00000100:00080000:0.0:1767320611.780874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320611.780877:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320611.780887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435b800 x1853163983113728/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320611.780894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320611.780897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320611.780898:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320611.780901:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5855 00000100:00080000:0.0:1767320611.780903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320611.780907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320611.780916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1c00 x1853163983113856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320611.780944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320611.780948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320611.780950:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320611.780953:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5855 00000100:00080000:0.0:1767320611.780955:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320611.780958:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320611.780967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163983114240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320611.780990:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983114112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320611.781065:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320611.781130:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320611.781137:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983113984:0@lo:400:kworker.0 00000100:00100000:0.0:1767320611.781159:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.781231:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114112 00000100:00100000:1.0:1767320611.781240:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114112 00000100:00100000:1.0:1767320611.781297:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113984 00000100:00100000:1.0:1767320611.781301:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113984 00000100:00100000:1.0:1767320611.781326:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121 00000100:00100000:1.0:1767320611.781335:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983114112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320611.781387:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983114112:12345-0@lo:400:kworker.0 Request processed in 53us (321us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320611.781394:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121 00000100:00100000:1.0:1767320611.781409:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122 00000100:00100000:1.0:1767320611.781414:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983113984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320611.781440:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983113984:12345-0@lo:400:kworker.0 Request processed in 26us (280us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320611.781446:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122 00000100:00100000:1.0:1767320611.781486:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983113984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320611.781496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983114112:0@lo:400:kworker.0 00000100:00100000:1.0:1767320611.781516:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983113472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781542:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.781564:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113472 00000100:00100000:1.0:1767320611.781567:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113472 00000100:00100000:1.0:1767320611.781587:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2405 00000100:00100000:1.0:1767320611.781592:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983113472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320611.781608:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320611.781634:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983113472:12345-0@lo:38:kworker.0 Request processed in 42us (92us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320611.781640:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2405 00000100:00080000:1.0:1767320611.781670:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320611.781674:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320611.781676:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983113472:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781684:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983113728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781704:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.781721:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113728 00000100:00100000:1.0:1767320611.781725:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113728 00000100:00100000:1.0:1767320611.781741:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2406 00000100:00100000:1.0:1767320611.781745:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983113728:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320611.781753:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320611.781773:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983113728:12345-0@lo:38:kworker.0 Request processed in 27us (69us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320611.781779:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2406 00000100:00080000:1.0:1767320611.781799:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320611.781803:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320611.781805:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983113728:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781811:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983113856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781833:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.781854:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113856 00000100:00100000:1.0:1767320611.781857:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113856 00000100:00100000:1.0:1767320611.781874:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 626 00000100:00100000:1.0:1767320611.781879:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983113856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320611.781888:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320611.781909:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983113856:12345-0@lo:38:kworker.0 Request processed in 30us (76us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320611.781915:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 626 00000100:00080000:1.0:1767320611.781941:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320611.781944:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320611.781948:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983113856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781953:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983114240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320611.781973:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320611.781989:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114240 00000100:00100000:1.0:1767320611.781992:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114240 00000100:00100000:1.0:1767320611.782008:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2407 00000100:00100000:1.0:1767320611.782012:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983114240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320611.782020:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320611.782043:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983114240:12345-0@lo:38:kworker.0 Request processed in 30us (69us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320611.782048:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2407 00000100:00080000:1.0:1767320611.782069:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320611.782073:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320611.782075:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983114240:0@lo:38:kworker.0 00000100:00100000:2.0:1767320611.782259:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983113344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320611.782323:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320611.782346:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320611.782354:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983113600:0@lo:400:kworker.0 00000100:00100000:2.0:1767320611.782377:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320611.782475:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113344 00000100:00100000:0.0:1767320611.782480:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113344 00000100:00100000:0.0:1767320611.782520:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1818 00000100:00100000:0.0:1767320611.782527:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983113344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320611.782561:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983113344:12345-0@lo:400:kworker.0 Request processed in 35us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320611.782567:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1818 00000100:00100000:0.0:1767320611.782578:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983113600 00000100:00100000:0.0:1767320611.782581:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983113600 00000100:00100000:0.0:1767320611.782599:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1819 00000100:00100000:0.0:1767320611.782604:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983113600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320611.782627:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983113600:12345-0@lo:400:kworker.0 Request processed in 24us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320611.782632:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1819 00000100:00080000:2.0:1767320611.782843:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320611.782871:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b100 x1853163983113600/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320627 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320611.782882:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983113600:0@lo:400:kworker.0 00010000:00080000:2.0:1767320611.782891:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604358000 x1853163983113344/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320627 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320611.782898:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983113344:0@lo:400:kworker.0 00010000:00010000:3.0:1767320611.975922:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320612.996494:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767320613.773396:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915691264 offset 0 length 520: 4 00000100:00100000:3.0:1767320613.777480:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320613.777619:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691520 00000100:00100000:1.0:1767320613.777630:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691520 00000100:00100000:1.0:1767320613.777711:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1820 00000100:00100000:1.0:1767320613.777731:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915691520:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320613.777805:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915691520:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320613.777813:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1820 00000100:00080000:2.0:1767320613.778847:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320613.780867:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320613.781012:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691392 00000100:00100000:0.0:1767320613.781021:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691392 00000100:00100000:0.0:1767320613.781088:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2408 00000100:00100000:0.0:1767320613.781097:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915691392:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320613.781120:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320613.781179:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915691392:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320613.781184:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2408 00010000:00010000:1.0:1767320614.025968:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320615.046160:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320616.077690:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320616.389483:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320616.389503:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983114368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320616.389568:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320616.389730:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114368 00000100:00100000:0.0:1767320616.389739:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114368 00000100:00100000:0.0:1767320616.389801:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1218 00000100:00100000:0.0:1767320616.389810:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983114368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320616.389842:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320616.389900:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983114368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320616.389908:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1218 00000100:00100000:2.0:1767320616.390164:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320616.390174:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983114496:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320616.390211:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320616.390339:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114496 00000100:00100000:1.0:1767320616.390344:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114496 00000100:00100000:1.0:1767320616.390376:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1219 00000100:00100000:1.0:1767320616.390382:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983114496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320616.390398:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320616.390443:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983114496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320616.390449:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1219 00000100:00100000:2.0:1767320616.390523:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983114496:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320616.390616:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320616.390628:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320616.390650:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983114368:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320616.900766:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320591/real 1767320591] req@ffff9346063b2300 x1853163983108608/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320616 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320616.900799:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346063b2300 x1853163983108608/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320616 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320616.900818:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320616.900827:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320616.900830:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983108608:0@lo:250:kworker.0 00000100:00080000:3.0:1767320616.901361:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320616.901367:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320616.901370:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320616.901374:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5839 00000100:00080000:3.0:1767320616.901378:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320616.901384:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320616.901415:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626d80 x1853163983114624/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320616.902019:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983114752:0@lo:400:kworker.0 00000100:00100000:2.0:1767320616.902314:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320616.902405:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320616.902409:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320616.902411:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320616.902416:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5860 00000100:00080000:3.0:1767320616.902419:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320616.902424:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320616.902443:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983114880/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320616.902566:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114752 00000100:00100000:1.0:1767320616.902574:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114752 00000100:00100000:1.0:1767320616.902709:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1821 00000100:00100000:1.0:1767320616.902720:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983114752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320616.902882:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983114624:0@lo:250:kworker.0 00000400:00000100:0.0:1767320616.902908:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983114624 offset 0 length 520: 4 00000100:00100000:0.0:1767320616.902926:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983114880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320616.902946:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320616.902965:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983114880 00000100:00100000:0.0:1767320616.902968:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983114880 00000100:00100000:0.0:1767320616.902991:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2409 00000100:00100000:0.0:1767320616.903010:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983114880:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320616.903029:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320616.903060:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983114880:12345-0@lo:38:kworker.0 Request processed in 62us (113us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320616.903066:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2409 00000100:00080000:0.0:1767320616.903101:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320616.903106:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320616.903109:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983114880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320616.903138:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983115008:0@lo:400:kworker.0 00000100:00100000:0.0:1767320616.903184:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320616.903210:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115008 00000100:00100000:0.0:1767320616.903214:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115008 00000100:00100000:0.0:1767320616.903237:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1822 00000100:00100000:0.0:1767320616.903244:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983115008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320616.903295:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983115008:12345-0@lo:400:kworker.0 Request processed in 52us (110us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320616.903302:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1822 00010000:00080000:0.0:1767320616.903329:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1180 x1853163983115008/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320632 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320616.903340:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983115008:0@lo:400:kworker.0 00000100:00080000:0.0:1767320616.903365:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320616.912301:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983114752:12345-0@lo:400:kworker.0 Request processed in 9571us (10116us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320616.912318:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1821 00000100:00080000:3.0:1767320616.917410:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320616.917420:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320616.917424:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320616.917442:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5860 00000100:00080000:3.0:1767320616.917453:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320616.917461:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320616.917506:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1180 x1853163983115136/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320616.917526:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320616.917530:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320616.917531:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320616.917535:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5860 00000100:00080000:3.0:1767320616.917537:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320616.917540:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320616.917555:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983115264/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320616.917785:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983115392:0@lo:400:kworker.0 00000100:00100000:1.0:1767320616.917865:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320616.917970:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115392 00000100:00100000:1.0:1767320616.917977:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115392 00000100:00100000:1.0:1767320616.918030:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1123 00000100:00100000:1.0:1767320616.918038:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983115392:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320616.918103:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983115392:12345-0@lo:400:kworker.0 Request processed in 66us (237us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320616.918111:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1123 00000100:00100000:1.0:1767320616.918159:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983115392:0@lo:400:kworker.0 00000100:00080000:3.0:1767320616.918690:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320616.918695:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320616.918698:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320616.918702:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5860 00000100:00080000:3.0:1767320616.918705:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320616.918710:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320616.918725:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163983115648/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320616.918753:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320616.918761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983115520:0@lo:400:kworker.0 00000100:00100000:3.0:1767320616.918791:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320616.918872:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115520 00000100:00100000:1.0:1767320616.918877:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115520 00000100:00100000:1.0:1767320616.918909:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124 00000100:00100000:1.0:1767320616.918916:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983115520:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320616.918953:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983115520:12345-0@lo:400:kworker.0 Request processed in 38us (161us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320616.918959:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124 00000100:00100000:1.0:1767320616.918990:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983115520:0@lo:400:kworker.0 00010000:00080000:2.0:1767320616.919352:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f2680 x1853163983114752/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320632 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320616.919364:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983114752:0@lo:400:kworker.0 00000100:00100000:0.0:1767320616.920883:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983115136:0@lo:38:kworker.0 00000100:00100000:0.0:1767320616.920942:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320616.920981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983115264:0@lo:38:kworker.0 00000100:00100000:0.0:1767320616.921019:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320616.921046:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983115648:0@lo:38:kworker.0 00000100:00100000:0.0:1767320616.921081:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320616.921359:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115136 00000100:00100000:2.0:1767320616.921367:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115136 00000100:00100000:2.0:1767320616.921403:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115648 00000100:00100000:2.0:1767320616.921407:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115648 00000100:00100000:2.0:1767320616.921425:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2410 00000100:00100000:2.0:1767320616.921442:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983115136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320616.921469:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320616.921512:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983115136:12345-0@lo:38:kworker.0 Request processed in 79us (569us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320616.921518:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2410 00000100:00100000:2.0:1767320616.921533:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2411 00000100:00100000:2.0:1767320616.921538:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983115648:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320616.921547:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320616.921568:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983115648:12345-0@lo:38:kworker.0 Request processed in 29us (486us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320616.921574:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2411 00000100:00100000:3.0:1767320616.921950:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115264 00000100:00100000:3.0:1767320616.921956:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115264 00000100:00100000:3.0:1767320616.921990:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 627 00000100:00100000:3.0:1767320616.922007:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983115264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320616.922021:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320616.922050:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983115264:12345-0@lo:38:kworker.0 Request processed in 53us (1030us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320616.922055:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 627 00000100:00080000:0.0:1767320616.922395:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320616.922401:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320616.922406:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983115136:0@lo:38:kworker.0 00000100:00080000:0.0:1767320616.922415:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320616.922419:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320616.922422:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983115264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320616.922429:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320616.922451:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320616.922454:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983115648:0@lo:38:kworker.0 00010000:00010000:1.0:1767320617.096537:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320618.116506:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320618.892493:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320618.892533:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320618.892560:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691648 00000100:00100000:0.0:1767320618.892571:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691648 00000100:00100000:0.0:1767320618.892615:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691776 00000100:00100000:0.0:1767320618.892619:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691776 00000100:00100000:0.0:1767320618.892666:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1823 00000100:00100000:0.0:1767320618.892681:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915691776:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320618.892681:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2412 00000100:00100000:1.0:1767320618.892693:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915691648:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320618.892714:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320618.892765:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915691776:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320618.892773:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1823 00000100:00080000:2.0:1767320618.892781:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320618.906998:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915691648:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 14294us (14501us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320618.907013:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2412 00010000:00010000:1.0:1767320619.146039:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320620.166535:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320621.188745:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320621.509635:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320621.509659:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983115776:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320621.509739:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320621.509801:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115776 00000100:00100000:2.0:1767320621.509810:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115776 00000100:00100000:2.0:1767320621.509882:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220 00000100:00100000:2.0:1767320621.509892:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983115776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320621.509949:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320621.509984:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320621.509999:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983115776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320621.510006:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220 00000100:00100000:0.0:1767320621.510099:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320621.510109:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983115904:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320621.510138:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320621.510213:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983115904 00000100:00100000:3.0:1767320621.510218:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983115904 00000100:00100000:1.0:1767320621.510267:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221 00000100:00100000:1.0:1767320621.510293:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983115904:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320621.510349:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:1.0:1767320621.510399:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320621.510433:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983115904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 141us (294us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320621.510440:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221 00000100:00100000:0.0:1767320621.510459:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983115904:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320621.510499:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983115776:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320622.021461:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.021473:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320622.021533:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.021537:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.021541:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.021549:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.021555:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.021561:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320622.021583:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911f80 x1853163983116160/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.021616:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.021620:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.021622:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.021625:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.021628:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.021631:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320622.021641:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910a80 x1853163983116416/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.021650:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.021653:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.021654:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.021658:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.021661:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.021664:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320622.021674:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911c00 x1853163983116544/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320622.021682:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320622.021685:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320622.021687:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320622.021690:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5865 00000100:00080000:0.0:1767320622.021692:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320622.021695:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320622.021704:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910e00 x1853163983116672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320622.021775:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983116288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.021839:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320622.021886:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320622.021891:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983116032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.021923:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320622.021953:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116288 00000100:00100000:3.0:1767320622.021961:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116288 00000100:00100000:3.0:1767320622.022021:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116032 00000100:00100000:3.0:1767320622.022024:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116032 00000100:00100000:3.0:1767320622.022047:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824 00000100:00100000:3.0:1767320622.022068:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983116288:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.022128:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983116288:12345-0@lo:400:kworker.0 Request processed in 72us (287us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320622.022136:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824 00000100:00100000:3.0:1767320622.022151:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1825 00000100:00100000:3.0:1767320622.022156:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983116032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320622.022181:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983116032:12345-0@lo:400:kworker.0 Request processed in 25us (257us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320622.022186:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1825 00000100:00080000:3.0:1767320622.022207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320622.022229:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910700 x1853163983116032/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320638 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320622.022242:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983116032:0@lo:400:kworker.0 00010000:00080000:3.0:1767320622.022253:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a912d80 x1853163983116288/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320638 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320622.022261:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983116288:0@lo:400:kworker.0 00000100:00100000:2.0:1767320622.022499:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983116160:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.022541:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320622.022564:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983116416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.022584:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320622.022608:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983116544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.022630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320622.022645:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983116672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.022660:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320622.022685:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116544 00000100:00100000:2.0:1767320622.022689:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116544 00000100:00100000:2.0:1767320622.022709:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 628 00000100:00100000:2.0:1767320622.022714:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983116544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320622.022732:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320622.022761:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983116544:12345-0@lo:38:kworker.0 Request processed in 45us (129us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320622.022766:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 628 00000100:00080000:2.0:1767320622.022787:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320622.022791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320622.022793:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983116544:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.022819:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116160 00000100:00100000:2.0:1767320622.022822:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116160 00000100:00100000:2.0:1767320622.022835:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116416 00000100:00100000:2.0:1767320622.022837:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116416 00000100:00100000:2.0:1767320622.022842:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116672 00000100:00100000:2.0:1767320622.022844:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116672 00000100:00100000:2.0:1767320622.022853:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2413 00000100:00100000:2.0:1767320622.022856:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116160:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320622.022865:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320622.022890:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116160:12345-0@lo:38:kworker.0 Request processed in 32us (350us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320622.022895:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2413 00000100:00100000:2.0:1767320622.022916:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2414 00000100:00100000:2.0:1767320622.022920:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116416:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320622.022928:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320622.022948:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116416:12345-0@lo:38:kworker.0 Request processed in 27us (363us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320622.022953:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2414 00000100:00100000:2.0:1767320622.022961:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2415 00000100:00100000:2.0:1767320622.022965:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320622.022973:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1767320622.023173:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320622.023177:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320622.023181:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983116160:0@lo:38:kworker.0 00000100:00080000:3.0:1767320622.023189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320622.023192:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320622.023194:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983116416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320622.023623:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983116672:12345-0@lo:38:kworker.0 Request processed in 654us (959us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320622.023629:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2415 00000100:00080000:3.0:1767320622.032142:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320622.032157:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320622.032163:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983116672:0@lo:38:kworker.0 00010000:00010000:1.0:1767320622.212789:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320623.236523:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320624.016599:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320624.017849:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692032 00000100:00100000:1.0:1767320624.017862:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692032 00000100:00080000:3.0:1767320624.017933:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320624.017967:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1826 00000100:00100000:1.0:1767320624.017979:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320624.018059:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (1467us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320624.018534:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1826 00000100:00100000:2.0:1767320624.024568:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320624.026135:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915691904 00000100:00100000:1.0:1767320624.026150:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915691904 00000100:00100000:1.0:1767320624.026216:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2416 00000100:00100000:1.0:1767320624.026225:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915691904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320624.026247:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320624.026360:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915691904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 134us (1801us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320624.026373:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2416 00010000:00010000:3.0:1767320624.266722:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320625.285547:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320626.311055:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320626.628490:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320626.638050:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983116928:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320626.638358:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320626.638416:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983116800:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320626.638440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320626.638570:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116928 00000100:00100000:3.0:1767320626.638579:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116928 00000100:00100000:3.0:1767320626.638642:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983116800 00000100:00100000:3.0:1767320626.638647:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983116800 00000100:00100000:3.0:1767320626.638666:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1222 00000100:00100000:3.0:1767320626.638675:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983116928:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320626.638707:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320626.638759:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983116928:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (627us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320626.638766:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1222 00000100:00100000:3.0:1767320626.638781:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223 00000100:00100000:3.0:1767320626.638787:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983116800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320626.638801:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320626.638855:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983116800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320626.638860:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223 00000100:00100000:3.0:1767320626.638902:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983116928:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320626.638914:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983116800:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320626.639129:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320626.639218:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320627.140662:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.140672:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320627.140749:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.140753:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.140756:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.140764:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.140769:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.140775:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320627.140797:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911880 x1853163983117184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.140830:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.140834:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.140836:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.140840:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.140842:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320627.140845:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320627.140857:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912300 x1853163983117440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.140865:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.140868:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.140870:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.140873:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.140876:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1767320627.140880:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320627.140890:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913480 x1853163983117568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320627.140899:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320627.140902:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320627.140904:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320627.140907:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5870 00000100:00080000:0.0:1767320627.140909:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1767320627.140912:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320627.140921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911180 x1853163983117696/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320627.141045:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983117312:0@lo:400:kworker.0 00000100:00100000:2.0:1767320627.141118:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320627.141166:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320627.141172:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983117056:0@lo:400:kworker.0 00000100:00100000:2.0:1767320627.141194:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320627.141507:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117312 00000100:00100000:0.0:1767320627.141515:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117312 00000100:00100000:0.0:1767320627.141581:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117056 00000100:00100000:0.0:1767320627.141584:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117056 00000100:00100000:0.0:1767320627.141611:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1827 00000100:00100000:0.0:1767320627.141619:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983117312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320627.141676:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983117312:12345-0@lo:400:kworker.0 Request processed in 57us (557us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320627.141683:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1827 00000100:00100000:0.0:1767320627.141699:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828 00000100:00100000:0.0:1767320627.141705:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983117056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320627.141732:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983117056:12345-0@lo:400:kworker.0 Request processed in 27us (538us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320627.141738:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828 00000100:00080000:2.0:1767320627.141794:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320627.141838:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911500 x1853163983117056/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320643 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320627.141851:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983117056:0@lo:400:kworker.0 00010000:00080000:2.0:1767320627.141860:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a910380 x1853163983117312/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320643 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320627.141867:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983117312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320627.142156:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983117184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.142189:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.142227:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983117440:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.142247:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.142295:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983117568:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.142324:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320627.142358:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983117696:0@lo:38:kworker.0 00000100:00100000:3.0:1767320627.142378:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320627.142460:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117568 00000100:00100000:1.0:1767320627.142465:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117568 00000100:00100000:1.0:1767320627.142499:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 629 00000100:00100000:1.0:1767320627.142504:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983117568:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320627.142522:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320627.142560:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983117568:12345-0@lo:38:kworker.0 Request processed in 54us (234us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320627.142565:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 629 00000100:00100000:1.0:1767320627.142585:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117184 00000100:00100000:1.0:1767320627.142588:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117184 00000100:00100000:1.0:1767320627.142604:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117440 00000100:00100000:1.0:1767320627.142606:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117440 00000100:00100000:1.0:1767320627.142616:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117696 00000100:00100000:1.0:1767320627.142619:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117696 00000100:00100000:1.0:1767320627.142628:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2417 00000100:00100000:1.0:1767320627.142632:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320627.142641:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320627.142677:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117184:12345-0@lo:38:kworker.0 Request processed in 44us (488us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320627.142683:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2417 00000100:00100000:1.0:1767320627.142693:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2418 00000100:00100000:1.0:1767320627.142696:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117440:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320627.142704:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320627.142722:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117440:12345-0@lo:38:kworker.0 Request processed in 25us (475us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320627.142727:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2418 00000100:00100000:1.0:1767320627.142736:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2419 00000100:00100000:1.0:1767320627.142740:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117696:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320627.142747:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320627.142767:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983117696:12345-0@lo:38:kworker.0 Request processed in 25us (387us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320627.142775:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2419 00000100:00080000:3.0:1767320627.142990:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.142996:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.143000:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983117184:0@lo:38:kworker.0 00000100:00080000:3.0:1767320627.143024:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.143027:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.143029:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983117440:0@lo:38:kworker.0 00000100:00080000:3.0:1767320627.143038:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.143041:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.143044:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983117568:0@lo:38:kworker.0 00000100:00080000:3.0:1767320627.143051:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320627.143054:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320627.143056:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983117696:0@lo:38:kworker.0 00010000:00010000:3.0:1767320627.334116:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320628.358093:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320629.132765:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320629.132822:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320629.132914:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692288 00000100:00100000:2.0:1767320629.132925:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692288 00000100:00100000:2.0:1767320629.132995:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692160 00000100:00100000:2.0:1767320629.132998:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692160 00000100:00100000:2.0:1767320629.133013:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1829 00000100:00100000:2.0:1767320629.133022:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320629.133101:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (346us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320629.133108:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1829 00000100:00100000:2.0:1767320629.133123:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2420 00000100:00100000:2.0:1767320629.133127:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915692160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320629.133148:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320629.133180:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915692160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (357us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320629.133186:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2420 00000100:00080000:3.0:1767320629.133464:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320629.381573:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320630.409402:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320631.431326:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320631.749033:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983117824:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320631.749118:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320631.749386:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117824 00000100:00100000:3.0:1767320631.749394:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117824 00000100:00100000:3.0:1767320631.749586:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224 00000100:00100000:3.0:1767320631.749596:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983117824:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320631.749631:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320631.749692:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983117824:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (573us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320631.749701:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224 00000100:00100000:2.0:1767320631.749739:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983117824:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320631.750599:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320631.752930:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320631.752954:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983117952:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320631.753026:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320631.753221:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983117952 00000100:00100000:3.0:1767320631.753230:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983117952 00000100:00100000:3.0:1767320631.753315:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1225 00000100:00100000:3.0:1767320631.753324:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983117952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320631.753362:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320631.753615:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983117952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 291us (588us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320631.753623:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1225 00000100:00080000:0.0:1767320631.753682:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320631.754238:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983117952:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320632.261905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.261915:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320632.261965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.261969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.261972:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.261980:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.261984:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.261991:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320632.262011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d880 x1853163983118208/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.262034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.262038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.262039:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.262043:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.262046:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320632.262049:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320632.262072:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ea00 x1853163983118464/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.262082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.262092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.262095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.262098:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.262102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320632.262106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320632.262118:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f100 x1853163983118592/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320632.262127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320632.262131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320632.262133:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320632.262136:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5875 00000100:00080000:3.0:1767320632.262138:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767320632.262141:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320632.262154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c000 x1853163983118720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320632.262192:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983118208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320632.262254:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.262322:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983118464:0@lo:38:kworker.0 00000100:00100000:3.0:1767320632.262341:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.262365:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983118592:0@lo:38:kworker.0 00000100:00100000:3.0:1767320632.262385:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.262403:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983118720:0@lo:38:kworker.0 00000100:00100000:3.0:1767320632.262420:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320632.262448:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118592 00000100:00100000:3.0:1767320632.262454:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118592 00000100:00100000:3.0:1767320632.262482:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 630 00000100:00100000:3.0:1767320632.262489:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983118592:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320632.262508:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320632.262549:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983118592:12345-0@lo:38:kworker.0 Request processed in 58us (162us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320632.262556:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 630 00000100:00100000:3.0:1767320632.262574:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118208 00000100:00100000:3.0:1767320632.262578:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118208 00000100:00100000:3.0:1767320632.262592:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118464 00000100:00100000:3.0:1767320632.262594:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118464 00000100:00100000:3.0:1767320632.262602:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118720 00000100:00100000:3.0:1767320632.262604:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118720 00000100:00100000:3.0:1767320632.262613:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2421 00000100:00100000:3.0:1767320632.262617:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320632.262626:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320632.262646:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118208:12345-0@lo:38:kworker.0 Request processed in 28us (394us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320632.262651:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2421 00000100:00100000:3.0:1767320632.262659:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2422 00000100:00100000:3.0:1767320632.262662:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320632.262670:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320632.262686:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118464:12345-0@lo:38:kworker.0 Request processed in 22us (344us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320632.262690:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2422 00000100:00100000:3.0:1767320632.262697:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2423 00000100:00100000:3.0:1767320632.262700:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320632.262707:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320632.262765:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983118720:12345-0@lo:38:kworker.0 Request processed in 63us (344us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320632.262770:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2423 00000100:00080000:3.0:1767320632.262814:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320632.262819:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320632.262822:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983118208:0@lo:38:kworker.0 00000100:00080000:3.0:1767320632.262831:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320632.262834:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320632.262836:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983118464:0@lo:38:kworker.0 00000100:00080000:3.0:1767320632.262844:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320632.262847:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320632.262849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983118592:0@lo:38:kworker.0 00000100:00080000:3.0:1767320632.262856:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320632.262859:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320632.262861:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983118720:0@lo:38:kworker.0 00000100:00100000:1.0:1767320632.263302:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983118080:0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.263338:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320632.263373:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320632.263378:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983118336:0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.263398:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320632.263423:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118080 00000100:00100000:1.0:1767320632.263428:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118080 00000100:00100000:1.0:1767320632.263461:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1830 00000100:00100000:1.0:1767320632.263468:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983118080:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.263509:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983118080:12345-0@lo:400:kworker.0 Request processed in 42us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320632.263515:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1830 00000100:00100000:1.0:1767320632.263528:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118336 00000100:00100000:1.0:1767320632.263531:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118336 00000100:00100000:1.0:1767320632.263547:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1831 00000100:00100000:1.0:1767320632.263553:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983118336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320632.263582:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983118336:12345-0@lo:400:kworker.0 Request processed in 30us (182us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320632.263588:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1831 00010000:00080000:1.0:1767320632.263623:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d500 x1853163983118336/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320648 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320632.263639:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983118336:0@lo:400:kworker.0 00010000:00080000:1.0:1767320632.263649:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469e300 x1853163983118080/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320648 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320632.263657:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983118080:0@lo:400:kworker.0 00000100:00080000:0.0:1767320632.264202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320632.454298:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320633.477046:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320634.253360:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320634.253719:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692544 00000100:00100000:0.0:1767320634.253729:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692544 00000100:00100000:0.0:1767320634.253878:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1832 00000100:00100000:0.0:1767320634.253890:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915692544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320634.253968:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915692544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (615us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320634.253980:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1832 00000100:00080000:2.0:1767320634.254103:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320634.255578:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320634.255700:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692416 00000100:00100000:0.0:1767320634.255709:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692416 00000100:00100000:0.0:1767320634.255903:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2424 00000100:00100000:0.0:1767320634.255910:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915692416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320634.255931:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320634.256142:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915692416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 230us (569us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320634.256151:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2424 00010000:00010000:1.0:1767320634.501106:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320635.525528:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320636.548615:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320636.868662:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320636.868688:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983118848:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320636.868760:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320636.868893:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118848 00000100:00100000:1.0:1767320636.868902:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118848 00000100:00100000:1.0:1767320636.868986:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1226 00000100:00100000:1.0:1767320636.868996:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983118848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320636.869029:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320636.869090:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983118848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 96us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320636.869098:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1226 00000100:00100000:1.0:1767320636.869149:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983118848:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320636.869291:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320636.869941:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983118976:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320636.869991:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320636.870067:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983118976 00000100:00100000:2.0:1767320636.870082:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983118976 00000100:00100000:2.0:1767320636.870131:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227 00000100:00100000:2.0:1767320636.870139:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983118976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320636.870171:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320636.870234:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320636.872393:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983118976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 2245us (2391us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320636.872408:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227 00000100:00100000:1.0:1767320636.872520:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983118976:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320637.384583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320637.384594:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320637.384670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320637.384674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320637.384679:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320637.384688:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5880 00000100:00080000:2.0:1767320637.384694:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320637.384701:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320637.384726:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014700 x1853163983119232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320637.384757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320637.384760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320637.384762:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320637.384765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5880 00000100:00080000:2.0:1767320637.384775:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767320637.384779:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320637.384792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015180 x1853163983119488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320637.384800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320637.384803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320637.384805:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320637.384809:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5880 00000100:00080000:2.0:1767320637.384812:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320637.384815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320637.384825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016680 x1853163983119616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320637.384835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320637.384837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320637.384839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320637.384843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5880 00000100:00080000:2.0:1767320637.384845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767320637.384848:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320637.384857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014a80 x1853163983119744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320637.385025:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983119232:0@lo:38:kworker.0 00000100:00100000:3.0:1767320637.385108:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.385187:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983119488:0@lo:38:kworker.0 00000100:00100000:3.0:1767320637.385208:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.385226:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983119616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320637.385249:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.385293:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983119744:0@lo:38:kworker.0 00000100:00100000:3.0:1767320637.385324:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320637.385360:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119232 00000100:00100000:3.0:1767320637.385368:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119232 00000100:00100000:3.0:1767320637.385395:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119488 00000100:00100000:3.0:1767320637.385398:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119488 00000100:00100000:3.0:1767320637.385410:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119744 00000100:00100000:3.0:1767320637.385413:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119744 00000100:00100000:3.0:1767320637.385425:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2425 00000100:00100000:3.0:1767320637.385432:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320637.385451:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320637.385490:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119232:12345-0@lo:38:kworker.0 Request processed in 57us (381us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320637.385496:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2425 00000100:00100000:3.0:1767320637.385510:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2426 00000100:00100000:3.0:1767320637.385513:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320637.385521:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320637.385538:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119488:12345-0@lo:38:kworker.0 Request processed in 24us (330us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320637.385543:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2426 00000100:00100000:3.0:1767320637.385552:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2427 00000100:00100000:3.0:1767320637.385555:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119744:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320637.385563:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320637.385581:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983119744:12345-0@lo:38:kworker.0 Request processed in 24us (257us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320637.385589:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2427 00000100:00080000:3.0:1767320637.385616:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320637.385631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320637.385635:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983119232:0@lo:38:kworker.0 00000100:00080000:3.0:1767320637.385646:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320637.385649:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320637.385651:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983119488:0@lo:38:kworker.0 00000100:00080000:3.0:1767320637.385658:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320637.385661:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320637.385663:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983119744:0@lo:38:kworker.0 00000100:00100000:0.0:1767320637.385987:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983119360:0@lo:400:kworker.0 00000100:00100000:0.0:1767320637.386019:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320637.386051:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320637.386073:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983119104:0@lo:400:kworker.0 00000100:00100000:0.0:1767320637.386095:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320637.386363:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119360 00000100:00100000:1.0:1767320637.386369:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119360 00000100:00100000:1.0:1767320637.386412:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119104 00000100:00100000:1.0:1767320637.386416:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119104 00000100:00100000:1.0:1767320637.386431:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1833 00000100:00100000:1.0:1767320637.386439:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983119360:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320637.386491:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983119360:12345-0@lo:400:kworker.0 Request processed in 52us (471us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320637.386498:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1833 00000100:00100000:1.0:1767320637.386513:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1834 00000100:00100000:1.0:1767320637.386520:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983119104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320637.386548:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983119104:12345-0@lo:400:kworker.0 Request processed in 28us (451us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320637.386553:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1834 00000100:00100000:1.0:1767320637.386572:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983119616 00000100:00100000:1.0:1767320637.386576:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983119616 00000100:00100000:1.0:1767320637.386591:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 631 00000100:00100000:1.0:1767320637.386596:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983119616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320637.386607:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320637.386653:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983119616:12345-0@lo:38:kworker.0 Request processed in 56us (1402us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320637.386659:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 631 00010000:00080000:1.0:1767320637.386686:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015c00 x1853163983119104/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320653 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320637.386698:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983119104:0@lo:400:kworker.0 00010000:00080000:1.0:1767320637.386707:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163983119360/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320653 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320637.386715:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983119360:0@lo:400:kworker.0 00000100:00080000:0.0:1767320637.386981:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320637.387052:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320637.387057:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320637.387077:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983119616:0@lo:38:kworker.0 00010000:00010000:1.0:1767320637.572584:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320638.598687:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320639.376086:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320639.376132:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915692672 offset 0 length 520: 4 00000100:00100000:1.0:1767320639.376224:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320639.377130:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692928 00000100:00100000:3.0:1767320639.377145:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692928 00000100:00100000:3.0:1767320639.377244:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915692800 00000100:00100000:3.0:1767320639.377248:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915692800 00000100:00100000:3.0:1767320639.377313:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1835 00000100:00100000:3.0:1767320639.377325:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692928:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320639.377521:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915692928:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 198us (1450us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320639.377530:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1835 00000100:00100000:3.0:1767320639.377552:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2428 00000100:00100000:3.0:1767320639.377556:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915692800:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:0.0:1767320639.377762:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767320639.378037:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320639.378092:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915692800:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 533us (1867us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320639.378100:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2428 00010000:00010000:3.0:1767320639.622009:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320640.648382:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320641.476828:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320616/real 1767320616] req@ffff93463c626d80 x1853163983114624/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320641 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320641.476877:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c626d80 x1853163983114624/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320641 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320641.476916:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320641.476949:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320641.476954:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983114624:0@lo:250:kworker.0 00000100:00080000:2.0:1767320641.477101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320641.477109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320641.477113:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320641.477120:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5865 00000100:00080000:2.0:1767320641.477126:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320641.477137:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320641.477187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016680 x1853163983119872/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320641.477248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320641.477252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320641.477254:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320641.477259:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5885 00000100:00080000:2.0:1767320641.477263:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320641.477267:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320641.477314:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163983120128/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320641.477337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320641.477341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320641.477342:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320641.477346:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5885 00000100:00080000:2.0:1767320641.477349:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767320641.477352:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320641.477367:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015180 x1853163983120384/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320641.477376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320641.477379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320641.477381:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320641.477385:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5885 00000100:00080000:2.0:1767320641.477387:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320641.477393:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320641.477405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014700 x1853163983120512/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320641.477477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320641.477481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320641.477483:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320641.477487:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5885 00000100:00080000:2.0:1767320641.477489:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320641.477493:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320641.477503:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af3480 x1853163983120896/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320641.477623:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983120256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.477716:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320641.477783:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320641.477790:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983120000:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.477813:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320641.477848:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983120640:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.477881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320641.477912:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320641.477917:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983120768:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.477951:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320641.477984:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120256 00000100:00100000:1.0:1767320641.477992:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120256 00000100:00100000:1.0:1767320641.478047:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120000 00000100:00100000:1.0:1767320641.478049:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120000 00000100:00100000:1.0:1767320641.478073:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836 00000100:00100000:1.0:1767320641.478082:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983120256:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478187:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983120256:12345-0@lo:400:kworker.0 Request processed in 104us (470us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.478196:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836 00000100:00100000:1.0:1767320641.478217:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1837 00000100:00100000:1.0:1767320641.478224:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983120000:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478251:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983120000:12345-0@lo:400:kworker.0 Request processed in 28us (437us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.478258:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1837 00000100:00100000:1.0:1767320641.478379:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120640 00000100:00100000:1.0:1767320641.478383:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120640 00000100:00100000:1.0:1767320641.478414:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120768 00000100:00100000:1.0:1767320641.478417:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120768 00000100:00100000:1.0:1767320641.478435:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125 00000100:00100000:1.0:1767320641.478441:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983120640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478476:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983120640:12345-0@lo:400:kworker.0 Request processed in 36us (594us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.478482:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125 00000100:00100000:1.0:1767320641.478492:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1126 00000100:00100000:1.0:1767320641.478497:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983120768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478519:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983120768:12345-0@lo:400:kworker.0 Request processed in 22us (567us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.478524:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1126 00010000:00080000:1.0:1767320641.478581:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163983120000/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320657 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320641.478597:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983120000:0@lo:400:kworker.0 00010000:00080000:1.0:1767320641.478608:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014a80 x1853163983120256/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320657 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320641.478616:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983120256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478655:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983120768:0@lo:400:kworker.0 00000100:00100000:1.0:1767320641.478665:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983120640:0@lo:400:kworker.0 00000100:00100000:3.0:1767320641.479361:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983119872:0@lo:250:kworker.0 00000400:00000100:3.0:1767320641.479408:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983119872 offset 0 length 520: 4 00000100:00100000:3.0:1767320641.479443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983120128:0@lo:38:kworker.0 00000100:00100000:3.0:1767320641.479475:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320641.479801:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983120384:0@lo:38:kworker.0 00000100:00100000:3.0:1767320641.479828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320641.479851:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983120512:0@lo:38:kworker.0 00000100:00100000:3.0:1767320641.479880:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320641.479916:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983120896:0@lo:38:kworker.0 00000100:00100000:3.0:1767320641.479938:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320641.479977:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120128 00000100:00100000:3.0:1767320641.479986:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120128 00000100:00100000:3.0:1767320641.480019:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120384 00000100:00100000:3.0:1767320641.480021:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120384 00000100:00100000:3.0:1767320641.480029:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120896 00000100:00100000:3.0:1767320641.480031:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120896 00000100:00100000:3.0:1767320641.480044:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2429 00000100:00100000:3.0:1767320641.480051:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120128:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320641.480075:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320641.480124:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120128:12345-0@lo:38:kworker.0 Request processed in 72us (648us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320641.480131:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2429 00000100:00100000:3.0:1767320641.480148:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2430 00000100:00100000:3.0:1767320641.480152:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120384:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320641.480161:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320641.480185:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120384:12345-0@lo:38:kworker.0 Request processed in 31us (356us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320641.480191:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2430 00000100:00100000:3.0:1767320641.480202:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2431 00000100:00100000:3.0:1767320641.480206:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120896:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320641.480215:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320641.480236:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983120896:12345-0@lo:38:kworker.0 Request processed in 29us (297us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320641.480241:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2431 00000100:00100000:3.0:1767320641.480264:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983120512 00000100:00100000:3.0:1767320641.480267:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983120512 00000100:00100000:3.0:1767320641.480328:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 632 00000100:00100000:3.0:1767320641.480333:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983120512:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320641.480343:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320641.480367:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983120512:12345-0@lo:38:kworker.0 Request processed in 34us (486us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320641.480373:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 632 00000100:00080000:3.0:1767320641.480420:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320641.480426:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320641.480430:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983120128:0@lo:38:kworker.0 00000100:00080000:3.0:1767320641.480439:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320641.480442:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320641.480445:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983120384:0@lo:38:kworker.0 00000100:00080000:3.0:1767320641.480453:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320641.480456:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320641.480459:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983120512:0@lo:38:kworker.0 00000100:00080000:3.0:1767320641.480466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320641.480469:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320641.480472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983120896:0@lo:38:kworker.0 00000100:00080000:0.0:1767320641.480852:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320641.480860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320641.480863:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320641.668624:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320641.988714:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983121024:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320641.988820:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320641.989346:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121024 00000100:00100000:1.0:1767320641.989360:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121024 00000100:00100000:1.0:1767320641.989440:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1228 00000100:00100000:1.0:1767320641.989452:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983121024:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320641.989491:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320641.989647:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983121024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 197us (826us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320641.989656:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1228 00000100:00100000:0.0:1767320641.989768:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983121024:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320641.990265:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983121152:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320641.990347:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320641.991820:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121152 00000100:00100000:0.0:1767320641.991837:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121152 00000100:00100000:0.0:1767320641.991928:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229 00000100:00100000:0.0:1767320641.991940:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983121152:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320641.991979:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320641.992313:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983121152:12345-0@lo:13:osp-pre-0-1.0 Request processed in 374us (1964us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320641.992325:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229 00000100:00100000:2.0:1767320641.992471:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983121152:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320642.692696:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320643.717178:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320644.497104:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320644.497339:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320644.506136:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693184 00000100:00100000:3.0:1767320644.506137:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693056 00000100:00100000:3.0:1767320644.506163:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693056 00000100:00100000:2.0:1767320644.506164:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693184 00000100:00080000:0.0:1767320644.509723:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320644.510512:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1838 00000100:00100000:3.0:1767320644.510630:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915693184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320644.510730:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915693184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 117us (13642us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320644.510738:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1838 00000100:00100000:3.0:1767320644.510760:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2432 00000100:00100000:3.0:1767320644.510764:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693056:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320644.510784:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320644.510810:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (13472us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320644.510817:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2432 00010000:00010000:1.0:1767320644.740527:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320645.764532:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320646.596588:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320646.596599:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320646.596671:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320646.596675:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320646.596679:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320646.596686:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5890 00000100:00080000:3.0:1767320646.596692:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320646.596698:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320646.596755:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9c00 x1853163983121408/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320646.596791:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320646.596794:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320646.596796:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320646.596800:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5890 00000100:00080000:3.0:1767320646.596803:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767320646.596808:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320646.596819:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8e00 x1853163983121664/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320646.596828:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320646.596831:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320646.596833:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320646.596837:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5890 00000100:00080000:3.0:1767320646.596839:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320646.596843:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320646.596854:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8a80 x1853163983121792/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320646.596879:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320646.596882:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320646.596884:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320646.596888:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5890 00000100:00080000:3.0:1767320646.596890:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320646.596893:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320646.596903:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aad80 x1853163983122176/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320646.596985:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983121536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320646.597050:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320646.597093:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320646.597099:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983121280:0@lo:400:kworker.0 00000100:00100000:0.0:1767320646.597122:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320646.597191:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121536 00000100:00100000:3.0:1767320646.597198:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121536 00000100:00100000:1.0:1767320646.597301:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983121920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320646.597307:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983122048:0@lo:400:kworker.0 00000100:00100000:3.0:1767320646.597333:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121280 00000100:00100000:3.0:1767320646.597338:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121280 00000100:00100000:1.0:1767320646.597350:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320646.597391:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1839 00000100:00100000:1.0:1767320646.597401:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983121536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320646.597466:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320646.597475:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983121536:12345-0@lo:400:kworker.0 Request processed in 76us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320646.597483:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1839 00000100:00100000:2.0:1767320646.597496:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983121408:0@lo:38:kworker.0 00000100:00100000:1.0:1767320646.597505:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121920 00000100:00100000:1.0:1767320646.597509:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121920 00000100:00100000:2.0:1767320646.597519:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320646.597534:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983121664:0@lo:38:kworker.0 00000100:00100000:2.0:1767320646.597553:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320646.597569:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983121792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320646.597591:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320646.597608:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983122176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320646.597626:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320646.597654:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121408 00000100:00100000:2.0:1767320646.597658:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121408 00000100:00100000:2.0:1767320646.597672:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121664 00000100:00100000:2.0:1767320646.597675:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121664 00000100:00100000:2.0:1767320646.597682:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122176 00000100:00100000:2.0:1767320646.597684:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122176 00000100:00100000:2.0:1767320646.597692:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2433 00000100:00100000:2.0:1767320646.597696:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983121408:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320646.597714:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320646.597749:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983121408:12345-0@lo:38:kworker.0 Request processed in 51us (228us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320646.597755:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2433 00000100:00100000:2.0:1767320646.597767:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2434 00000100:00100000:2.0:1767320646.597771:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983121664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320646.597781:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320646.597802:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983121664:12345-0@lo:38:kworker.0 Request processed in 30us (248us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320646.597807:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2434 00000100:00100000:2.0:1767320646.597817:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2435 00000100:00100000:2.0:1767320646.597821:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983122176:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320646.597828:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320646.597850:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983122176:12345-0@lo:38:kworker.0 Request processed in 28us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320646.597855:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2435 00000100:00080000:2.0:1767320646.597886:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320646.597891:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320646.597895:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983121408:0@lo:38:kworker.0 00000100:00080000:2.0:1767320646.597903:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320646.597906:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320646.597909:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983121664:0@lo:38:kworker.0 00000100:00080000:2.0:1767320646.597916:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320646.597919:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320646.597922:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983122176:0@lo:38:kworker.0 00000100:00100000:2.0:1767320646.597967:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983121792 00000100:00100000:2.0:1767320646.597971:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983121792 00000100:00100000:2.0:1767320646.597991:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 633 00000100:00100000:2.0:1767320646.597995:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983121792:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320646.598005:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320646.598032:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983121792:12345-0@lo:38:kworker.0 Request processed in 35us (440us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320646.598037:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 633 00000100:00080000:2.0:1767320646.598056:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320646.598060:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320646.598063:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983121792:0@lo:38:kworker.0 00000100:00100000:2.0:1767320646.598102:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122048 00000100:00100000:2.0:1767320646.598105:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122048 00000100:00080000:0.0:1767320646.598162:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320646.598167:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320646.598169:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320646.598200:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aa680 x1853163983121536/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320662 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320646.598211:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983121536:0@lo:400:kworker.0 00000100:00100000:3.0:1767320646.598812:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840 00000100:00100000:3.0:1767320646.598822:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983121280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320646.598876:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983121280:12345-0@lo:400:kworker.0 Request processed in 56us (1754us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320646.598883:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840 00010000:00080000:3.0:1767320646.598913:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626d80 x1853163983121280/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320662 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320646.598922:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983121280:0@lo:400:kworker.0 00000100:00100000:1.0:1767320646.599300:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1767320646.599309:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983121920:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320646.599367:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983121920:12345-0@lo:400:kworker.0 Request processed in 60us (2016us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320646.599373:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1767320646.599629:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983121920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320646.600652:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128 00000100:00100000:2.0:1767320646.600665:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983122048:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320646.600742:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983122048:12345-0@lo:400:kworker.0 Request processed in 81us (3275us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320646.600749:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128 00000100:00100000:2.0:1767320646.600789:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983122048:0@lo:400:kworker.0 00010000:00010000:1.0:1767320646.798658:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320647.108697:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320647.108718:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983122304:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320647.108787:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320647.108887:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122304 00000100:00100000:1.0:1767320647.108895:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122304 00000100:00100000:1.0:1767320647.108945:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1230 00000100:00100000:1.0:1767320647.108953:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983122304:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320647.108982:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320647.109035:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983122304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320647.109043:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1230 00000100:00100000:0.0:1767320647.109189:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983122304:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320647.109435:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983122432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320647.109474:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320647.109492:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122432 00000100:00100000:2.0:1767320647.109497:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122432 00000100:00100000:2.0:1767320647.109578:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1231 00000100:00100000:2.0:1767320647.109587:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983122432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320647.109604:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320647.109656:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983122432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 71us (181us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320647.109662:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1231 00000100:00100000:0.0:1767320647.109721:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983122432:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320647.812728:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320648.836625:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320649.612534:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320649.613496:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320649.613670:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693440 00000100:00100000:3.0:1767320649.613681:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693440 00000100:00100000:3.0:1767320649.613761:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693312 00000100:00100000:3.0:1767320649.613765:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693312 00000100:00100000:3.0:1767320649.613785:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1841 00000100:00100000:3.0:1767320649.613795:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915693440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320649.613869:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915693440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (1347us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320649.613878:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1841 00000100:00100000:3.0:1767320649.613897:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2436 00000100:00100000:3.0:1767320649.613902:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320649.613923:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320649.613961:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (469us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320649.613968:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2436 00000100:00080000:0.0:1767320649.614226:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320649.860777:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320650.885342:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320651.716596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320651.716607:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320651.716710:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1767320651.716815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320651.716821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320651.716825:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320651.716834:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5895 00000100:00080000:3.0:1767320651.716839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320651.716845:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320651.716868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab480 x1853163983122688/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320651.716939:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983122688:0@lo:38:kworker.0 00000100:00100000:0.0:1767320651.717009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320651.717059:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122688 00000100:00100000:0.0:1767320651.717066:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122688 00000100:00100000:0.0:1767320651.717101:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2437 00000100:00100000:0.0:1767320651.717108:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983122688:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320651.717127:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320651.717165:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983122688:12345-0@lo:38:kworker.0 Request processed in 56us (155us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320651.717173:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2437 00000100:00080000:0.0:1767320651.717213:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320651.717217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320651.717222:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983122688:0@lo:38:kworker.0 00000100:00100000:1.0:1767320651.717401:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320651.717413:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983122816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320651.717446:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320651.717521:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122816 00000100:00100000:0.0:1767320651.717525:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122816 00000100:00100000:0.0:1767320651.717571:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1842 00000100:00100000:0.0:1767320651.717579:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983122816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320651.717620:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983122816:12345-0@lo:400:kworker.0 Request processed in 42us (173us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320651.717627:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1842 00010000:00080000:1.0:1767320651.717702:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a9500 x1853163983122816/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320667 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320651.717716:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983122816:0@lo:400:kworker.0 00000100:00080000:1.0:1767320651.717732:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320651.717969:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983122560:0@lo:400:kworker.0 00000100:00100000:2.0:1767320651.718001:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320651.718134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320651.718139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320651.718142:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320651.718146:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5895 00000100:00080000:3.0:1767320651.718149:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320651.718154:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320651.718173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8700 x1853163983122944/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320651.718192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320651.718196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320651.718198:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320651.718202:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5895 00000100:00080000:3.0:1767320651.718204:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320651.718208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320651.718219:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aaa00 x1853163983123072/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320651.718250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320651.718253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320651.718255:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320651.718258:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5895 00000100:00080000:3.0:1767320651.718262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320651.718264:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320651.718296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8380 x1853163983123456/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320651.718320:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983123328:0@lo:400:kworker.0 00000100:00100000:2.0:1767320651.718620:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983123200:0@lo:400:kworker.0 00000100:00100000:2.0:1767320651.718653:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320651.718709:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983122944:0@lo:38:kworker.0 00000100:00100000:2.0:1767320651.718734:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320651.718776:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123200 00000100:00100000:1.0:1767320651.718782:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123200 00000100:00100000:1.0:1767320651.718824:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1129 00000100:00100000:1.0:1767320651.718832:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983123200:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320651.718873:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983123200:12345-0@lo:400:kworker.0 Request processed in 42us (219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320651.718881:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1129 00000100:00080000:1.0:1767320651.718905:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320651.718967:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320651.719025:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983123200:0@lo:400:kworker.0 00000100:00100000:1.0:1767320651.719087:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123328 00000100:00100000:1.0:1767320651.719091:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123328 00000100:00100000:1.0:1767320651.719130:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130 00000100:00100000:1.0:1767320651.719137:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983123328:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320651.719172:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983123328:12345-0@lo:400:kworker.0 Request processed in 35us (204us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320651.719178:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130 00000100:00100000:1.0:1767320651.719206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983123328:0@lo:400:kworker.0 00000100:00080000:3.0:1767320651.719262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320651.719729:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983123072:0@lo:38:kworker.0 00000100:00100000:2.0:1767320651.719763:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320651.719805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983123456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320651.719831:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320651.719871:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123072 00000100:00100000:3.0:1767320651.719876:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123072 00000100:00100000:3.0:1767320651.719903:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 634 00000100:00100000:3.0:1767320651.719909:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983123072:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320651.719927:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320651.719958:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983123072:12345-0@lo:38:kworker.0 Request processed in 48us (194us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320651.719965:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 634 00000100:00100000:0.0:1767320651.720231:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122560 00000100:00100000:0.0:1767320651.720237:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122560 00000100:00100000:0.0:1767320651.720306:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983122944 00000100:00100000:0.0:1767320651.720310:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983122944 00000100:00100000:0.0:1767320651.720346:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1843 00000100:00100000:0.0:1767320651.720354:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983122560:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320651.720419:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983122560:12345-0@lo:400:kworker.0 Request processed in 66us (2416us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320651.720426:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1843 00000100:00100000:0.0:1767320651.720440:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2438 00000100:00100000:0.0:1767320651.720445:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983122944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320651.720456:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320651.720480:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983122944:12345-0@lo:38:kworker.0 Request processed in 34us (1745us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320651.720487:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2438 00010000:00080000:0.0:1767320651.720522:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aa300 x1853163983122560/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320667 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320651.720534:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983122560:0@lo:400:kworker.0 00000100:00080000:3.0:1767320651.720605:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320651.721015:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320651.721022:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320651.721026:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983122944:0@lo:38:kworker.0 00000100:00080000:2.0:1767320651.721035:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320651.721038:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320651.721041:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983123072:0@lo:38:kworker.0 00000100:00100000:1.0:1767320651.721090:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123456 00000100:00100000:1.0:1767320651.721095:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123456 00000100:00100000:1.0:1767320651.721162:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2439 00000100:00100000:1.0:1767320651.721168:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983123456:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320651.721180:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320651.721265:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983123456:12345-0@lo:38:kworker.0 Request processed in 95us (1433us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320651.721289:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2439 00000100:00080000:2.0:1767320651.721602:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320651.721607:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320651.721611:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983123456:0@lo:38:kworker.0 00010000:00010000:3.0:1767320651.908704:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320652.229361:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983123584:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320652.229434:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320652.230391:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123584 00000100:00100000:1.0:1767320652.230403:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123584 00000100:00100000:1.0:1767320652.230470:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232 00000100:00100000:1.0:1767320652.230480:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983123584:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320652.230522:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320652.230589:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983123584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 109us (1155us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320652.230598:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232 00000100:00100000:1.0:1767320652.230649:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983123584:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320652.233819:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983123712:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320652.233895:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320652.234030:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123712 00000100:00100000:2.0:1767320652.234039:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123712 00000100:00100000:2.0:1767320652.234099:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233 00000100:00100000:2.0:1767320652.234108:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983123712:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320652.234141:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320652.234198:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983123712:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (303us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320652.234206:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233 00000100:00100000:0.0:1767320652.234527:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983123712:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320652.933567:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320653.956567:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320654.735691:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320654.735808:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320654.735998:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693696 00000100:00100000:3.0:1767320654.736009:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693696 00000100:00100000:3.0:1767320654.736097:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693568 00000100:00100000:3.0:1767320654.736101:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693568 00000100:00100000:3.0:1767320654.736124:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1844 00000100:00100000:3.0:1767320654.736135:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915693696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320654.736208:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915693696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (527us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320654.736220:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1844 00000100:00100000:3.0:1767320654.736237:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2440 00000100:00100000:3.0:1767320654.736242:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693568:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320654.736262:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320654.736357:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915693568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 113us (549us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320654.736363:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2440 00000100:00080000:0.0:1767320654.736479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320654.981220:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320656.004900:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320656.836417:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320656.836429:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320656.836483:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320656.836488:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320656.836498:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320656.836506:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5900 00000100:00080000:0.0:1767320656.836511:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320656.836516:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320656.836537:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1880 x1853163983123968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320656.836555:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320656.836558:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320656.836560:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320656.836563:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5900 00000100:00080000:0.0:1767320656.836566:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320656.836569:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320656.836576:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3480 x1853163983124224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320656.836584:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320656.836586:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320656.836588:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320656.836591:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5900 00000100:00080000:0.0:1767320656.836593:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320656.836596:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320656.836604:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1500 x1853163983124352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320656.836639:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320656.836642:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320656.836644:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320656.836647:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983124480:0@lo:400:kworker.0 00000100:00080000:0.0:1767320656.836658:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5900 00000100:00080000:0.0:1767320656.836662:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320656.836665:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320656.836680:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3100 x1853163983124736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320656.836704:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983124608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320656.836728:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320656.836761:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983124096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320656.836791:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320656.836816:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320656.836817:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124480 00000100:00100000:0.0:1767320656.836823:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983123840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320656.836825:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124480 00000100:00100000:0.0:1767320656.836845:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320656.836875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320656.836896:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124608 00000100:00100000:1.0:1767320656.836901:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124608 00000100:00100000:1.0:1767320656.836927:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1131 00000100:00100000:1.0:1767320656.836936:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983124480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320656.836985:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983124480:12345-0@lo:400:kworker.0 Request processed in 50us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320656.836992:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1131 00000100:00100000:1.0:1767320656.837009:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1132 00000100:00100000:1.0:1767320656.837016:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983124608:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320656.837040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320656.837045:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320656.837054:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983124608:12345-0@lo:400:kworker.0 Request processed in 38us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320656.837058:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124096 00000100:00100000:1.0:1767320656.837063:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1132 00000100:00100000:0.0:1767320656.837064:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124096 00000100:00100000:0.0:1767320656.837095:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123840 00000100:00100000:0.0:1767320656.837098:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123840 00000100:00100000:0.0:1767320656.837118:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1845 00000100:00100000:0.0:1767320656.837125:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983124096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320656.837130:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983124608:0@lo:400:kworker.0 00000100:00100000:1.0:1767320656.837166:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983124480:0@lo:400:kworker.0 00000100:00100000:0.0:1767320656.837170:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983124096:12345-0@lo:400:kworker.0 Request processed in 43us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320656.837178:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1845 00000100:00080000:1.0:1767320656.837191:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320656.837195:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1846 00000100:00100000:0.0:1767320656.837202:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983123840:12345-0@lo:400:kworker.0 00010000:00080000:1.0:1767320656.837218:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e2300 x1853163983124096/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320672 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320656.837229:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983124096:0@lo:400:kworker.0 00000100:00100000:0.0:1767320656.837236:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983123840:12345-0@lo:400:kworker.0 Request processed in 32us (388us total) trans 0 rc 0/0 00010000:00080000:1.0:1767320656.837250:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e2d80 x1853163983123840/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320672 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320656.837259:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983123840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320656.837355:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1846 00000100:00100000:2.0:1767320656.837483:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983123968:0@lo:38:kworker.0 00000100:00100000:2.0:1767320656.837516:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320656.837546:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983124224:0@lo:38:kworker.0 00000100:00100000:2.0:1767320656.837568:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320656.837591:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983124352:0@lo:38:kworker.0 00000100:00100000:2.0:1767320656.837613:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320656.837642:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983124736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320656.837716:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983123968 00000100:00100000:3.0:1767320656.837721:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983123968 00000100:00100000:3.0:1767320656.837742:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124224 00000100:00100000:3.0:1767320656.837745:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124224 00000100:00100000:3.0:1767320656.837762:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2441 00000100:00100000:3.0:1767320656.837767:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983123968:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320656.837784:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320656.837817:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983123968:12345-0@lo:38:kworker.0 Request processed in 48us (300us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320656.837822:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2441 00000100:00100000:3.0:1767320656.837833:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2442 00000100:00100000:3.0:1767320656.837837:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983124224:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320656.837845:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320656.837866:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983124224:12345-0@lo:38:kworker.0 Request processed in 28us (298us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320656.837872:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2442 00000100:00100000:2.0:1767320656.838047:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320656.838083:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320656.838088:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320656.838091:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983123968:0@lo:38:kworker.0 00000100:00080000:2.0:1767320656.838100:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320656.838104:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320656.838109:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983124224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320656.838125:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124352 00000100:00100000:1.0:1767320656.838131:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124352 00000100:00100000:1.0:1767320656.838172:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 635 00000100:00100000:1.0:1767320656.838178:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983124352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320656.838189:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320656.838221:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983124352:12345-0@lo:38:kworker.0 Request processed in 42us (606us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320656.838227:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 635 00000100:00080000:2.0:1767320656.838386:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320656.838391:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320656.838394:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983124352:0@lo:38:kworker.0 00000100:00100000:0.0:1767320656.838461:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124736 00000100:00100000:0.0:1767320656.838465:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124736 00000100:00100000:0.0:1767320656.838502:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2443 00000100:00100000:0.0:1767320656.838507:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983124736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320656.838517:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320656.838556:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983124736:12345-0@lo:38:kworker.0 Request processed in 48us (509us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320656.838562:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2443 00000100:00080000:1.0:1767320656.838606:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320656.838612:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320656.838616:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983124736:0@lo:38:kworker.0 00010000:00010000:1.0:1767320657.028509:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320657.348731:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983124864:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320657.348802:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320657.350009:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124864 00000100:00100000:1.0:1767320657.350031:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124864 00000100:00100000:1.0:1767320657.350100:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1234 00000100:00100000:1.0:1767320657.350110:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983124864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320657.350142:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320657.351266:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983124992:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320657.351339:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320657.352925:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983124992 00000100:00100000:0.0:1767320657.352938:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983124992 00000100:00100000:0.0:1767320657.353002:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767320657.353012:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983124992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320657.353053:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320657.353126:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983124992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 115us (1786us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320657.353135:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235 00000100:00100000:0.0:1767320657.353188:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983124992:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320657.353956:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983124864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 3845us (5151us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320657.353965:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1234 00000100:00100000:2.0:1767320657.355979:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983124864:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320658.058895:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320659.077039:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320659.856885:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320659.856935:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320659.860204:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693952 00000100:00100000:2.0:1767320659.860230:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693952 00000100:00100000:2.0:1767320659.860332:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915693824 00000100:00100000:2.0:1767320659.860336:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915693824 00000100:00100000:2.0:1767320659.860371:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1847 00000100:00100000:2.0:1767320659.860381:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915693952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767320659.860504:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320659.861985:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2444 00000100:00100000:3.0:1767320659.861995:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915693824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320659.862018:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320659.862290:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915693824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 293us (5353us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320659.862298:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2444 00000100:00100000:2.0:1767320659.862915:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915693952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 2534us (6322us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320659.862924:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1847 00010000:00010000:1.0:1767320660.100869:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320661.125485:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320661.963363:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320661.963385:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320661.963453:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320661.963457:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320661.963461:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320661.963469:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5905 00000100:00080000:0.0:1767320661.963474:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320661.963481:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320661.963504:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0e00 x1853163983125248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320661.963541:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320661.963544:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320661.963546:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320661.963550:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5905 00000100:00080000:0.0:1767320661.963561:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320661.963566:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320661.963578:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3b80 x1853163983125504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320661.963589:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320661.963593:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320661.963594:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320661.963600:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5905 00000100:00100000:3.0:1767320661.963600:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983125248:0@lo:38:kworker.0 00000100:00080000:0.0:1767320661.963605:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320661.963611:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320661.963623:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3100 x1853163983125632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320661.963659:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320661.963663:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320661.963665:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320661.963670:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5905 00000100:00080000:0.0:1767320661.963672:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320661.963679:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320661.963679:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320661.963694:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1880 x1853163983126016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320661.963735:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983125888:0@lo:400:kworker.0 00000100:00100000:3.0:1767320661.963749:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983125504:0@lo:38:kworker.0 00000100:00100000:0.0:1767320661.963776:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320661.963785:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320661.963810:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320661.963819:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983125632:0@lo:38:kworker.0 00000100:00100000:0.0:1767320661.963821:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983125760:0@lo:400:kworker.0 00000100:00100000:3.0:1767320661.963858:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320661.963897:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983126016:0@lo:38:kworker.0 00000100:00100000:1.0:1767320661.963905:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983125120:0@lo:400:kworker.0 00000100:00100000:3.0:1767320661.963922:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320661.963947:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320661.963967:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125888 00000100:00100000:3.0:1767320661.963976:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125888 00000100:00100000:1.0:1767320661.963979:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320661.963985:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983125376:0@lo:400:kworker.0 00000100:00100000:1.0:1767320661.964008:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320661.964045:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125248 00000100:00100000:3.0:1767320661.964045:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1133 00000100:00100000:1.0:1767320661.964051:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125248 00000100:00100000:0.0:1767320661.964053:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320661.964068:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125504 00000100:00100000:1.0:1767320661.964072:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125504 00000100:00100000:3.0:1767320661.964073:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983125888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320661.964080:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126016 00000100:00100000:1.0:1767320661.964084:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126016 00000100:00100000:0.0:1767320661.964090:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125760 00000100:00100000:1.0:1767320661.964090:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125120 00000100:00100000:0.0:1767320661.964093:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125760 00000100:00100000:1.0:1767320661.964093:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125120 00000100:00100000:1.0:1767320661.964111:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125376 00000100:00100000:1.0:1767320661.964114:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125376 00000100:00100000:1.0:1767320661.964132:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848 00000100:00100000:0.0:1767320661.964134:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1134 00000100:00100000:3.0:1767320661.964140:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983125888:12345-0@lo:400:kworker.0 Request processed in 82us (361us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320661.964141:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983125120:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320661.964142:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983125760:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320661.964156:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1133 00000100:00100000:0.0:1767320661.964195:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983125760:12345-0@lo:400:kworker.0 Request processed in 52us (141us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320661.964195:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983125120:12345-0@lo:400:kworker.0 Request processed in 55us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320661.964203:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1849 00000100:00100000:0.0:1767320661.964207:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1134 00000100:00100000:3.0:1767320661.964216:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983125376:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320661.964253:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983125376:12345-0@lo:400:kworker.0 Request processed in 40us (243us total) trans 0 rc 0/0 00010000:00080000:0.0:1767320661.964617:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0a80 x1853163983125376/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320677 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320661.964629:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983125376:0@lo:400:kworker.0 00010000:00080000:0.0:1767320661.964640:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e2680 x1853163983125120/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320677 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320661.964648:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983125120:0@lo:400:kworker.0 00000100:00100000:1.0:1767320661.971657:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848 00000100:00100000:1.0:1767320661.971765:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2445 00000100:00100000:1.0:1767320661.971774:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983125248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320661.971796:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320661.971891:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983125248:12345-0@lo:38:kworker.0 Request processed in 117us (8215us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320661.971898:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2445 00000100:00100000:1.0:1767320661.971914:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2446 00000100:00100000:1.0:1767320661.971919:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983125504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320661.971929:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320661.971955:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983125504:12345-0@lo:38:kworker.0 Request processed in 34us (8171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320661.971961:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2446 00000100:00100000:1.0:1767320661.971972:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2447 00000100:00100000:1.0:1767320661.971977:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983126016:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320661.971986:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320661.972012:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983126016:12345-0@lo:38:kworker.0 Request processed in 33us (8089us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320661.972018:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2447 00000100:00080000:1.0:1767320661.972034:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320661.972040:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320661.972042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320661.972076:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320661.972082:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320661.972086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983126016:0@lo:38:kworker.0 00000100:00080000:1.0:1767320661.972097:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320661.972100:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320661.972103:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983125248:0@lo:38:kworker.0 00000100:00080000:1.0:1767320661.972111:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320661.972114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320661.972117:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983125504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320661.972200:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983125760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320661.972215:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983125888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320661.972363:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983125632 00000100:00100000:1.0:1767320661.972370:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983125632 00000100:00100000:1.0:1767320661.972402:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 636 00000100:00100000:1.0:1767320661.972406:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983125632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320661.972417:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320661.972466:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983125632:12345-0@lo:38:kworker.0 Request processed in 57us (8607us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320661.972472:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 636 00000100:00080000:1.0:1767320661.977660:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320661.977675:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320661.977680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983125632:0@lo:38:kworker.0 00000100:00100000:3.0:1767320661.982032:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1849 00010000:00010000:1.0:1767320662.151130:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320662.468608:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983126144:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320662.468682:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320662.468913:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983126272:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320662.468941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320662.468983:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126144 00000100:00100000:1.0:1767320662.468989:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126144 00000100:00100000:1.0:1767320662.469026:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126272 00000100:00100000:1.0:1767320662.469029:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126272 00000100:00100000:1.0:1767320662.469044:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236 00000100:00100000:1.0:1767320662.469064:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983126144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320662.469094:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320662.469131:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983126144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 68us (449us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320662.469137:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236 00000100:00100000:1.0:1767320662.469153:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1237 00000100:00100000:1.0:1767320662.469171:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983126272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320662.469184:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320662.469212:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983126272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320662.469218:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1237 00000100:00100000:1.0:1767320662.469257:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983126272:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320662.469394:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983126144:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320663.178336:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320663.948701:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320663.948803:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915694080 offset 0 length 520: 4 00000100:00100000:1.0:1767320663.948853:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320663.949867:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694336 00000100:00100000:3.0:1767320663.949879:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694336 00000100:00100000:3.0:1767320663.949961:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694208 00000100:00100000:3.0:1767320663.949965:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694208 00000100:00100000:3.0:1767320663.949990:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1850 00000100:00100000:3.0:1767320663.950000:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915694336:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320663.950099:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915694336:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 99us (1408us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320663.950107:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1850 00000100:00100000:3.0:1767320663.950129:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2448 00000100:00100000:3.0:1767320663.950146:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915694208:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320663.950171:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320663.950201:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915694208:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (1347us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320663.950208:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2448 00000100:00080000:1.0:1767320663.950738:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320664.198832:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320665.220592:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320666.052696:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320641/real 1767320641] req@ffff934633016680 x1853163983119872/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320666 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320666.052830:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934633016680 x1853163983119872/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320666 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320666.052846:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320666.052914:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320666.052919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983119872:0@lo:250:kworker.0 00000100:00080000:0.0:1767320666.056156:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320666.056169:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320666.056174:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320666.056182:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5890 00000100:00080000:0.0:1767320666.056187:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320666.056195:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320666.056606:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e0a80 x1853163983126400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320666.056741:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983126528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320666.056825:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320666.057007:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320666.057011:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320666.057013:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320666.057014:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126528 00000100:00080000:0.0:1767320666.057017:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5910 00000100:00080000:0.0:1767320666.057020:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320666.057022:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126528 00000100:00080000:0.0:1767320666.057024:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320666.057041:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e2a00 x1853163983126656/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320666.057080:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1851 00000100:00100000:3.0:1767320666.057090:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983126528:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320666.057113:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320666.057128:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983126400:0@lo:250:kworker.0 00000400:00000100:1.0:1767320666.057153:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983126400 offset 0 length 520: 4 00000100:00100000:1.0:1767320666.057173:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983126656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320666.057191:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320666.057246:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126656 00000100:00100000:1.0:1767320666.057251:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126656 00000100:00100000:1.0:1767320666.057293:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2449 00000100:00100000:1.0:1767320666.057299:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983126656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320666.057317:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320666.057351:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983126656:12345-0@lo:38:kworker.0 Request processed in 51us (158us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320666.057358:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2449 00000100:00080000:1.0:1767320666.057408:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320666.057412:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320666.057415:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983126656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320666.057452:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983126784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320666.057475:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320666.057500:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126784 00000100:00100000:1.0:1767320666.057503:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126784 00000100:00100000:1.0:1767320666.057528:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852 00000100:00100000:1.0:1767320666.057533:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983126784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320666.057561:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983126784:12345-0@lo:400:kworker.0 Request processed in 28us (85us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320666.057569:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852 00000100:00080000:1.0:1767320666.057584:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320666.057599:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e3100 x1853163983126784/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320682 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320666.057611:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983126784:0@lo:400:kworker.0 00000100:00080000:0.0:1767320666.058375:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320666.058391:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320666.058394:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320666.058398:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5910 00000100:00080000:0.0:1767320666.058402:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320666.058406:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320666.058431:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e3480 x1853163983126912/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320666.058448:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320666.058451:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320666.058453:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320666.058457:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5910 00000100:00080000:0.0:1767320666.058460:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320666.058464:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320666.058475:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1500 x1853163983127040/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320666.058509:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320666.058512:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320666.058514:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320666.058518:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5910 00000100:00080000:0.0:1767320666.058520:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320666.058524:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320666.058534:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463a8e1880 x1853163983127424/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320666.058564:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983127168:0@lo:400:kworker.0 00000100:00100000:0.0:1767320666.058613:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320666.058640:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320666.058645:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983127296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320666.058665:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320666.058700:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127168 00000100:00100000:0.0:1767320666.058705:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127168 00000100:00100000:0.0:1767320666.059065:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127296 00000100:00100000:0.0:1767320666.059070:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127296 00000100:00100000:0.0:1767320666.059111:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1135 00000100:00100000:0.0:1767320666.059119:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983127168:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320666.059174:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983127168:12345-0@lo:400:kworker.0 Request processed in 55us (560us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320666.059181:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1135 00000100:00100000:0.0:1767320666.059197:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1136 00000100:00100000:0.0:1767320666.059203:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983127296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320666.059229:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983127296:12345-0@lo:400:kworker.0 Request processed in 26us (562us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320666.059247:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1136 00000100:00100000:0.0:1767320666.059312:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983127296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320666.059321:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983127168:0@lo:400:kworker.0 00000100:00080000:0.0:1767320666.059340:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320666.059343:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320666.059637:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983126912:0@lo:38:kworker.0 00000100:00100000:1.0:1767320666.059676:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320666.059705:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983127040:0@lo:38:kworker.0 00000100:00100000:1.0:1767320666.059814:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320666.059839:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983127424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320666.059861:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320666.059892:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127040 00000100:00100000:1.0:1767320666.059897:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127040 00000100:00100000:1.0:1767320666.059921:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 637 00000100:00100000:1.0:1767320666.059926:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983127040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320666.060046:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320666.060085:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983127040:12345-0@lo:38:kworker.0 Request processed in 157us (358us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320666.060103:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 637 00000100:00100000:1.0:1767320666.060127:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983126912 00000100:00100000:1.0:1767320666.060131:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983126912 00000100:00100000:1.0:1767320666.060148:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127424 00000100:00100000:1.0:1767320666.060151:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127424 00000100:00100000:1.0:1767320666.060164:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2450 00000100:00100000:1.0:1767320666.060168:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983126912:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320666.060179:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320666.060204:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983126912:12345-0@lo:38:kworker.0 Request processed in 35us (529us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320666.060210:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2450 00000100:00100000:1.0:1767320666.060220:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2451 00000100:00100000:1.0:1767320666.060223:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983127424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320666.060555:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320666.060586:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983127424:12345-0@lo:38:kworker.0 Request processed in 361us (723us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320666.060592:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2451 00000100:00080000:1.0:1767320666.060628:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320666.060632:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320666.060635:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983126912:0@lo:38:kworker.0 00000100:00080000:1.0:1767320666.060642:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320666.060645:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320666.060647:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983127040:0@lo:38:kworker.0 00000100:00080000:1.0:1767320666.060653:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320666.060655:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320666.060658:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983127424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320666.063453:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983126528:12345-0@lo:400:kworker.0 Request processed in 6351us (6615us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320666.063473:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1851 00010000:00080000:2.0:1767320666.063671:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463a8e0700 x1853163983126528/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320682 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320666.063688:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983126528:0@lo:400:kworker.0 00010000:00010000:1.0:1767320666.244822:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320667.269473:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320667.588786:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320667.588815:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983127552:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320667.588879:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320667.588958:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127552 00000100:00100000:0.0:1767320667.588966:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127552 00000100:00100000:0.0:1767320667.589020:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1238 00000100:00100000:0.0:1767320667.589038:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983127552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320667.589074:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320667.589114:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983127552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320667.589121:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1238 00000100:00100000:1.0:1767320667.589550:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983127552:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320667.589676:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983127680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320667.589702:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320667.589763:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127680 00000100:00100000:2.0:1767320667.589767:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127680 00000100:00100000:2.0:1767320667.589796:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239 00000100:00100000:2.0:1767320667.589803:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983127680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320667.589819:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320667.589856:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983127680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (152us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320667.589862:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239 00000100:00100000:1.0:1767320667.589916:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983127680:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320668.294515:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320669.072246:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320669.072331:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320669.072423:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694464 00000100:00100000:1.0:1767320669.072433:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694464 00000100:00100000:1.0:1767320669.072479:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694592 00000100:00100000:1.0:1767320669.072483:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694592 00000100:00100000:1.0:1767320669.072543:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1853 00000100:00100000:1.0:1767320669.072553:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915694592:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320669.072636:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915694592:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (303us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320669.072644:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1853 00000100:00100000:1.0:1767320669.072664:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2452 00000100:00100000:1.0:1767320669.072669:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915694464:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320669.072688:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320669.072714:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915694464:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (478us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320669.072721:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2452 00000100:00080000:0.0:1767320669.073220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320669.316537:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320670.340567:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320671.177763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320671.177784:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320671.177845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320671.177849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320671.177853:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320671.177862:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5914 00000100:00080000:2.0:1767320671.177867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320671.177873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320671.177893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7100 x1853163983127936/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320671.177917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320671.177920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320671.177922:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320671.177925:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5914 00000100:00080000:2.0:1767320671.177927:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320671.177931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320671.177940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4380 x1853163983128192/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320671.177948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320671.177950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320671.177952:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320671.177955:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5914 00000100:00080000:2.0:1767320671.177957:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320671.177960:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320671.177968:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee4000 x1853163983128320/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320671.177994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320671.177996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320671.178029:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320671.178033:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5914 00000100:00080000:2.0:1767320671.178035:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320671.178038:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320671.178048:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163983128704/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320671.178073:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983128576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320671.178148:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320671.178195:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320671.178202:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983128448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320671.178225:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320671.178255:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128576 00000100:00100000:2.0:1767320671.178305:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128576 00000100:00100000:2.0:1767320671.178355:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128448 00000100:00100000:2.0:1767320671.178358:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128448 00000100:00100000:1.0:1767320671.178748:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983128064:0@lo:400:kworker.0 00000100:00100000:1.0:1767320671.178793:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.178816:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320671.178822:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983127808:0@lo:400:kworker.0 00000100:00100000:1.0:1767320671.178843:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.178869:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128064 00000100:00100000:1.0:1767320671.178873:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128064 00000100:00100000:1.0:1767320671.178903:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127808 00000100:00100000:1.0:1767320671.178905:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127808 00000100:00100000:1.0:1767320671.178926:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1854 00000100:00100000:1.0:1767320671.178933:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983128064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320671.178979:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983128064:12345-0@lo:400:kworker.0 Request processed in 46us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320671.178988:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1854 00000100:00100000:1.0:1767320671.179018:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1855 00000100:00100000:1.0:1767320671.179024:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983127808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320671.179051:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983127808:12345-0@lo:400:kworker.0 Request processed in 27us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320671.179056:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1855 00010000:00080000:1.0:1767320671.179091:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6a00 x1853163983127808/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320687 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320671.179104:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983127808:0@lo:400:kworker.0 00010000:00080000:1.0:1767320671.179113:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5f80 x1853163983128064/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320687 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320671.179120:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983128064:0@lo:400:kworker.0 00000100:00100000:1.0:1767320671.179157:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983127936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320671.179179:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.179194:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983128192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320671.179211:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.179223:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983128320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320671.179261:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.179299:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983128704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320671.179339:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320671.179368:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983127936 00000100:00100000:1.0:1767320671.179619:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983127936 00000100:00100000:1.0:1767320671.179636:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128192 00000100:00100000:1.0:1767320671.179639:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128192 00000100:00100000:1.0:1767320671.179645:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128704 00000100:00100000:1.0:1767320671.179647:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128704 00000100:00100000:1.0:1767320671.179655:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2453 00000100:00100000:1.0:1767320671.179659:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983127936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320671.179677:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320671.179711:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983127936:12345-0@lo:38:kworker.0 Request processed in 51us (530us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320671.179717:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2453 00000100:00100000:1.0:1767320671.179728:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2454 00000100:00100000:1.0:1767320671.179732:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983128192:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320671.179739:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320671.179757:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983128192:12345-0@lo:38:kworker.0 Request processed in 24us (545us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320671.179762:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2454 00000100:00100000:1.0:1767320671.179770:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2455 00000100:00100000:1.0:1767320671.179773:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983128704:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320671.179781:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320671.179798:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983128704:12345-0@lo:38:kworker.0 Request processed in 23us (459us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320671.179803:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2455 00000100:00100000:1.0:1767320671.179817:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128320 00000100:00100000:1.0:1767320671.179820:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128320 00000100:00100000:1.0:1767320671.179836:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 638 00000100:00100000:1.0:1767320671.179839:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983128320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320671.179847:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320671.179867:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983128320:12345-0@lo:38:kworker.0 Request processed in 27us (618us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320671.179872:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 638 00000100:00080000:1.0:1767320671.179900:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320671.179905:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320671.179907:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983127936:0@lo:38:kworker.0 00000100:00080000:1.0:1767320671.179914:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320671.179917:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320671.179919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983128192:0@lo:38:kworker.0 00000100:00080000:1.0:1767320671.179925:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320671.179928:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320671.179931:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983128320:0@lo:38:kworker.0 00000100:00080000:1.0:1767320671.179937:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320671.179940:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320671.179942:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983128704:0@lo:38:kworker.0 00000100:00100000:2.0:1767320671.180308:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1137 00000100:00100000:2.0:1767320671.180319:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983128576:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320671.180362:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983128576:12345-0@lo:400:kworker.0 Request processed in 47us (2214us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320671.180368:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1137 00000100:00100000:2.0:1767320671.180791:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1138 00000100:00100000:2.0:1767320671.180801:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983128448:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320671.180832:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983128448:12345-0@lo:400:kworker.0 Request processed in 35us (2607us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320671.180837:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1138 00000100:00100000:2.0:1767320671.180873:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983128448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320671.180883:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983128576:0@lo:400:kworker.0 00000100:00080000:0.0:1767320671.181209:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320671.181215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320671.181217:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320671.364517:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320672.388601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320672.709072:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1767320672.709097:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983128832:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320672.709174:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320672.709221:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983128960:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320672.709248:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320672.709375:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128832 00000100:00100000:3.0:1767320672.709385:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128832 00000100:00100000:3.0:1767320672.709440:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983128960 00000100:00100000:3.0:1767320672.709443:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983128960 00000100:00100000:3.0:1767320672.709460:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1240 00000100:00100000:3.0:1767320672.709469:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983128832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320672.709501:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320672.709560:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983128832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (385us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320672.709567:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1240 00000100:00100000:3.0:1767320672.709583:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241 00000100:00100000:3.0:1767320672.709589:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324c2c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983128960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320672.709602:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320672.709627:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324c2c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983128960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320672.709633:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241 00000100:00100000:2.0:1767320672.709886:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983128832:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320672.709899:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983128960:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320673.414110:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320674.188384:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320674.188448:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320674.188529:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694848 00000100:00100000:2.0:1767320674.188543:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694848 00000100:00100000:2.0:1767320674.188639:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694720 00000100:00100000:2.0:1767320674.188642:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694720 00000100:00100000:2.0:1767320674.188664:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1856 00000100:00100000:2.0:1767320674.188675:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915694848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320674.188755:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915694848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (388us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320674.188767:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1856 00000100:00100000:2.0:1767320674.188790:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2456 00000100:00100000:2.0:1767320674.188795:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915694720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320674.188814:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320674.188839:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915694720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (391us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320674.188845:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2456 00000100:00080000:0.0:1767320674.188910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320674.436508:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320675.460745:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320676.294883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320676.294893:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320676.295013:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983129088:0@lo:400:kworker.0 00000100:00100000:1.0:1767320676.295096:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320676.295207:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129088 00000100:00100000:3.0:1767320676.295215:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129088 00000100:00100000:3.0:1767320676.295318:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1857 00000100:00100000:3.0:1767320676.295328:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983129088:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320676.295444:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320676.295734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320676.295739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320676.295743:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320676.295749:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5919 00000100:00080000:2.0:1767320676.295753:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320676.295758:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320676.295782:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7b80 x1853163983129216/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320676.295812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320676.295815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320676.295816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320676.295819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5919 00000100:00080000:2.0:1767320676.295820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320676.295823:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320676.295830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6d80 x1853163983129472/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320676.295836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320676.295838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320676.295839:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320676.295841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5919 00000100:00080000:2.0:1767320676.295842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320676.295844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320676.295850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5880 x1853163983129600/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320676.295895:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983129856:0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.295938:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.295980:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129856 00000100:00100000:0.0:1767320676.295985:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129856 00000100:00100000:0.0:1767320676.296031:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767320676.296039:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983129856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296074:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983129856:12345-0@lo:400:kworker.0 Request processed in 36us (135us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320676.296081:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1139 00000100:00100000:0.0:1767320676.296107:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983129728:0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296130:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.296160:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320676.296167:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983129344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296240:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.296263:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129728 00000100:00100000:0.0:1767320676.296267:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129728 00000100:00100000:0.0:1767320676.296324:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1140 00000100:00100000:0.0:1767320676.296330:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983129728:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296367:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983129728:12345-0@lo:400:kworker.0 Request processed in 37us (236us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320676.296373:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1140 00000100:00100000:0.0:1767320676.296393:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129344 00000100:00100000:0.0:1767320676.296399:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129344 00000100:00100000:0.0:1767320676.296422:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1858 00000100:00100000:0.0:1767320676.296426:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983129344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296458:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983129344:12345-0@lo:400:kworker.0 Request processed in 31us (218us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320676.296464:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1858 00000100:00080000:0.0:1767320676.296479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320676.296482:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320676.296485:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320676.296511:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983129856:0@lo:400:kworker.0 00010000:00080000:0.0:1767320676.296543:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5180 x1853163983129344/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320692 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320676.296552:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983129344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296561:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983129728:0@lo:400:kworker.0 00000100:00100000:0.0:1767320676.296588:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983129216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320676.296612:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.296628:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983129472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320676.296650:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.296667:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983129600:0@lo:38:kworker.0 00000100:00100000:0.0:1767320676.296693:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320676.296729:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129216 00000100:00100000:0.0:1767320676.296732:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129216 00000100:00100000:0.0:1767320676.296745:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129472 00000100:00100000:0.0:1767320676.296747:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129472 00000100:00100000:0.0:1767320676.296755:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2457 00000100:00100000:0.0:1767320676.296759:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983129216:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320676.296775:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320676.296800:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983129216:12345-0@lo:38:kworker.0 Request processed in 40us (187us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320676.296806:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2457 00000100:00100000:0.0:1767320676.296815:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2458 00000100:00100000:0.0:1767320676.296818:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983129472:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320676.296824:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320676.296840:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983129472:12345-0@lo:38:kworker.0 Request processed in 21us (190us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320676.296844:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2458 00000100:00080000:0.0:1767320676.296861:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320676.296865:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320676.296868:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983129216:0@lo:38:kworker.0 00000100:00080000:0.0:1767320676.296874:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320676.296876:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320676.296879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983129472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320676.296914:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129600 00000100:00100000:0.0:1767320676.296917:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129600 00000100:00100000:0.0:1767320676.296934:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 639 00000100:00100000:0.0:1767320676.296938:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983129600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320676.296946:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320676.296970:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983129600:12345-0@lo:38:kworker.0 Request processed in 30us (276us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320676.296975:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 639 00000100:00080000:0.0:1767320676.296999:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320676.297002:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320676.297005:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983129600:0@lo:38:kworker.0 00000100:00080000:2.0:1767320676.298175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320676.298182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320676.298186:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320676.298191:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5919 00000100:00080000:2.0:1767320676.298195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320676.298200:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320676.298225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6680 x1853163983129984/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320676.298551:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983129984:0@lo:38:kworker.0 00000100:00100000:0.0:1767320676.298598:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320676.298784:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983129984 00000100:00100000:2.0:1767320676.298791:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983129984 00000100:00100000:2.0:1767320676.298827:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2459 00000100:00100000:2.0:1767320676.298833:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983129984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320676.298851:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320676.298899:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983129984:12345-0@lo:38:kworker.0 Request processed in 64us (299us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320676.298916:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2459 00000100:00080000:1.0:1767320676.298991:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320676.298996:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320676.298999:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983129984:0@lo:38:kworker.0 00010000:00080000:2.0:1767320676.300004:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5c00 x1853163983129088/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320692 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320676.300018:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983129088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320676.300239:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983129088:12345-0@lo:400:kworker.0 Request processed in 4912us (5142us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320676.300249:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1857 00010000:00010000:3.0:1767320676.484654:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320677.508506:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320677.828639:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983130112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320677.828711:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320677.829841:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130112 00000100:00100000:2.0:1767320677.829851:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130112 00000100:00100000:2.0:1767320677.829922:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1242 00000100:00100000:2.0:1767320677.829932:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983130112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320677.829963:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320677.830259:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983130112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (1315us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320677.830268:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1242 00000100:00100000:2.0:1767320677.830363:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983130112:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320677.830551:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320677.830561:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983130240:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320677.830593:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320677.830622:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130240 00000100:00100000:0.0:1767320677.830625:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130240 00000100:00100000:0.0:1767320677.830644:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1243 00000100:00100000:0.0:1767320677.830648:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983130240:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320677.830663:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320677.830684:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983130240:12345-0@lo:13:osp-pre-0-1.0 Request processed in 36us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320677.830688:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1243 00000100:00100000:0.0:1767320677.830710:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983130240:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320678.541745:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320679.309159:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320679.309454:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695104 00000100:00100000:3.0:1767320679.309464:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695104 00000100:00100000:3.0:1767320679.309553:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1859 00000100:00100000:3.0:1767320679.309565:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915695104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767320679.309575:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320679.311046:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915695104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 1476us (1889us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320679.311058:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1859 00000100:00100000:0.0:1767320679.313716:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320679.313930:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915694976 00000100:00100000:3.0:1767320679.313941:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915694976 00000100:00100000:3.0:1767320679.313993:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2460 00000100:00100000:3.0:1767320679.314002:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915694976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320679.314026:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320679.314548:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915694976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 544us (835us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320679.314557:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2460 00010000:00010000:3.0:1767320679.556817:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320680.580447:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320681.412488:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320681.412499:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320681.412578:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320681.412582:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320681.412586:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320681.412594:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5924 00000100:00080000:0.0:1767320681.412599:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320681.412605:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320681.412629:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f0000 x1853163983130496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320681.412676:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320681.412680:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320681.412682:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320681.412686:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5924 00000100:00080000:0.0:1767320681.412688:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320681.412692:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320681.412704:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1c00 x1853163983130752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320681.412712:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320681.412715:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320681.412716:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320681.412719:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5924 00000100:00080000:0.0:1767320681.412722:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320681.412725:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320681.412738:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2d80 x1853163983130880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320681.412757:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320681.412760:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320681.412762:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320681.412765:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5924 00000100:00080000:0.0:1767320681.412768:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320681.412770:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320681.412780:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f1880 x1853163983131264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320681.412804:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983130624:0@lo:400:kworker.0 00000100:00100000:0.0:1767320681.412884:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320681.412947:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320681.412953:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983131008:0@lo:400:kworker.0 00000100:00100000:0.0:1767320681.412978:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320681.413023:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131008 00000100:00100000:0.0:1767320681.413031:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131008 00000100:00100000:0.0:1767320681.413082:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1141 00000100:00100000:0.0:1767320681.413091:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983131008:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320681.413149:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983131008:12345-0@lo:400:kworker.0 Request processed in 59us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320681.413156:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1141 00000100:00100000:0.0:1767320681.413200:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983131008:0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413327:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983131136:0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413356:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320681.413380:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320681.413385:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983130368:0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413403:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320681.413432:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130624 00000100:00100000:1.0:1767320681.413435:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130624 00000100:00100000:1.0:1767320681.413460:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130368 00000100:00100000:1.0:1767320681.413463:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130368 00000100:00100000:1.0:1767320681.413482:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860 00000100:00100000:1.0:1767320681.413488:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983130624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413517:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983130624:12345-0@lo:400:kworker.0 Request processed in 29us (633us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320681.413523:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860 00000100:00100000:1.0:1767320681.413535:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1861 00000100:00100000:1.0:1767320681.413541:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983130368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413567:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983130368:12345-0@lo:400:kworker.0 Request processed in 27us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320681.413573:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1861 00000100:00080000:1.0:1767320681.413591:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320681.413595:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:1.0:1767320681.413613:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543f800 x1853163983130368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320697 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320681.413623:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983130368:0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413643:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131136 00000100:00100000:1.0:1767320681.413646:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131136 00000100:00100000:1.0:1767320681.413668:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1142 00000100:00100000:1.0:1767320681.413673:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983131136:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320681.413700:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983131136:12345-0@lo:400:kworker.0 Request processed in 27us (344us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320681.413706:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1142 00000100:00080000:1.0:1767320681.413717:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320681.413730:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983131136:0@lo:400:kworker.0 00010000:00080000:0.0:1767320681.413821:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f1500 x1853163983130624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320697 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320681.413830:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983130624:0@lo:400:kworker.0 00000100:00100000:3.0:1767320681.414642:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983130496:0@lo:38:kworker.0 00000100:00100000:3.0:1767320681.414698:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320681.414750:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983130752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320681.414757:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130496 00000100:00100000:0.0:1767320681.414764:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130496 00000100:00100000:3.0:1767320681.414777:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320681.417401:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983130880:0@lo:38:kworker.0 00000100:00100000:0.0:1767320681.417479:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130752 00000100:00100000:3.0:1767320681.417490:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320681.417491:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130752 00000100:00100000:3.0:1767320681.417526:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983131264:0@lo:38:kworker.0 00000100:00100000:0.0:1767320681.417558:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2461 00000100:00100000:3.0:1767320681.417560:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320681.417571:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983130496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320681.417596:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320681.417606:0:71013:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983130880 00000100:00100000:0.0:1767320681.417650:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983130496:12345-0@lo:38:kworker.0 Request processed in 79us (2951us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320681.417657:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2461 00000100:00100000:0.0:1767320681.417675:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131264 00000100:00100000:0.0:1767320681.417679:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131264 00000100:00100000:0.0:1767320681.417691:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2462 00000100:00100000:0.0:1767320681.417696:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983130752:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320681.417706:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320681.417730:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983130752:12345-0@lo:38:kworker.0 Request processed in 33us (2953us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320681.417736:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2462 00000100:00100000:0.0:1767320681.417747:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2463 00000100:00100000:0.0:1767320681.417751:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983131264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320681.417759:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320681.417785:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983131264:12345-0@lo:38:kworker.0 Request processed in 32us (230us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320681.417790:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2463 00000100:00100000:3.0:1767320681.417839:0:71013:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983130880 00000100:00100000:3.0:1767320681.417874:0:71013:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 640 00000100:00100000:0.0:1767320681.417879:0:71014:0:(service.c:3434:ptlrpc_start_thread()) ost_out[0] started 2 min 2 max 16 00000100:00100000:3.0:1767320681.417880:0:71013:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983130880:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320681.417891:0:71014:0:(service.c:3493:ptlrpc_start_thread()) starting thread 'll_ost_out00_002' 00010000:00020000:3.0:1767320681.417895:0:71013:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320681.417928:0:71013:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983130880:12345-0@lo:38:kworker.0 Request processed in 47us (438us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320681.417934:0:71013:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 640 00000100:00080000:3.0:1767320681.417977:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320681.417983:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320681.417987:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983130496:0@lo:38:kworker.0 00000100:00080000:3.0:1767320681.418000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320681.418004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320681.418007:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983130752:0@lo:38:kworker.0 00000100:00080000:3.0:1767320681.418015:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320681.418019:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320681.418022:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983130880:0@lo:38:kworker.0 00000100:00080000:3.0:1767320681.418030:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320681.418035:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320681.418038:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983131264:0@lo:38:kworker.0 00010000:00010000:1.0:1767320681.605499:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320682.628598:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320682.949361:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320682.949365:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983131520:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320682.949373:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983131392:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320682.949446:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320682.949523:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131392 00000100:00100000:3.0:1767320682.949535:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131392 00000100:00100000:3.0:1767320682.949600:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244 00000100:00100000:3.0:1767320682.949609:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983131392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320682.949677:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320682.954506:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320682.957732:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131520 00000100:00100000:0.0:1767320682.957749:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131520 00000100:00100000:0.0:1767320682.958005:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245 00000100:00100000:0.0:1767320682.958017:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983131520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320682.958054:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320682.958142:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983131520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 128us (3642us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320682.958150:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245 00000100:00100000:2.0:1767320682.960430:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983131520:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320682.966326:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983131392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 16705us (16867us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320682.966343:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244 00000100:00100000:1.0:1767320682.977406:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983131392:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320683.656301:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320684.428099:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320684.428302:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695360 00000100:00100000:2.0:1767320684.428312:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695360 00000100:00100000:2.0:1767320684.428435:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1862 00000100:00100000:2.0:1767320684.428446:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915695360:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320684.428531:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915695360:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (444us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320684.428539:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1862 00000100:00100000:3.0:1767320684.432252:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320684.432448:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695232 00000100:00100000:0.0:1767320684.432459:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695232 00000100:00100000:0.0:1767320684.432509:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2464 00000100:00100000:0.0:1767320684.432518:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695232:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320684.432551:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320684.432621:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695232:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (380us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320684.432630:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2464 00000100:00080000:1.0:1767320684.434660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320684.676518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320685.701101:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320686.535818:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320686.535831:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320686.536410:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767320686.536569:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320686.536574:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320686.536579:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320686.536587:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5929 00000100:00080000:1.0:1767320686.536593:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320686.536600:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320686.536634:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163983131776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320686.536689:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320686.536692:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320686.536694:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320686.536698:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5929 00000100:00080000:1.0:1767320686.536700:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320686.536704:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320686.536725:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163983132032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320686.536734:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320686.536737:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320686.536739:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320686.536742:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5929 00000100:00080000:1.0:1767320686.536745:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320686.536748:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320686.536762:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f480 x1853163983132160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320686.536823:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983131904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.536892:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320686.536962:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320686.536968:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983132288:0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.536996:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320686.537057:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132288 00000100:00100000:0.0:1767320686.537064:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132288 00000100:00100000:0.0:1767320686.537129:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1143 00000100:00100000:0.0:1767320686.537138:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983132288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.537188:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983132288:12345-0@lo:400:kworker.0 Request processed in 51us (190us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320686.537195:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1143 00000100:00080000:0.0:1767320686.537215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320686.537245:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983132288:0@lo:400:kworker.0 00000100:00100000:3.0:1767320686.540096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983131776:0@lo:38:kworker.0 00000100:00100000:3.0:1767320686.540182:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320686.540238:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983132032:0@lo:38:kworker.0 00000100:00100000:3.0:1767320686.540260:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320686.540313:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983132160:0@lo:38:kworker.0 00000100:00100000:3.0:1767320686.540343:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320686.540389:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131904 00000100:00100000:3.0:1767320686.540396:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131904 00000100:00100000:3.0:1767320686.540459:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131776 00000100:00100000:3.0:1767320686.540463:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131776 00000100:00100000:3.0:1767320686.540479:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132032 00000100:00100000:3.0:1767320686.540482:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132032 00000100:00100000:3.0:1767320686.540494:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1863 00000100:00100000:3.0:1767320686.540503:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983131904:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320686.540567:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983131904:12345-0@lo:400:kworker.0 Request processed in 65us (3675us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320686.540575:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1863 00000100:00100000:3.0:1767320686.540593:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2465 00000100:00100000:3.0:1767320686.540597:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983131776:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320686.540628:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320686.540663:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983131776:12345-0@lo:38:kworker.0 Request processed in 64us (480us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320686.540670:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2465 00000100:00100000:3.0:1767320686.540682:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2466 00000100:00100000:3.0:1767320686.540687:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983132032:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320686.540695:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320686.540726:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983132032:12345-0@lo:38:kworker.0 Request processed in 29us (455us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320686.540732:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2466 00000100:00080000:3.0:1767320686.540751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320686.540775:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320686.540780:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320686.540784:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983131776:0@lo:38:kworker.0 00000100:00080000:3.0:1767320686.540792:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320686.540795:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320686.540797:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983132032:0@lo:38:kworker.0 00010000:00080000:3.0:1767320686.540829:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d880 x1853163983131904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320702 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320686.540841:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983131904:0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.542056:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132160 00000100:00100000:0.0:1767320686.542065:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132160 00000100:00100000:0.0:1767320686.542117:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 641 00000100:00100000:0.0:1767320686.542124:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983132160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320686.542138:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320686.542194:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983132160:12345-0@lo:38:kworker.0 Request processed in 69us (1850us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320686.542200:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 641 00000100:00080000:0.0:1767320686.542231:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320686.542235:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320686.542238:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983132160:0@lo:38:kworker.0 00000100:00100000:2.0:1767320686.542835:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983132416:0@lo:400:kworker.0 00000100:00100000:2.0:1767320686.542877:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320686.542897:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983131648:0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.543021:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132416 00000100:00100000:0.0:1767320686.543027:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132416 00000100:00100000:0.0:1767320686.543070:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1144 00000100:00100000:0.0:1767320686.543079:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983132416:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320686.543257:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320686.543262:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320686.543265:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320686.543284:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5929 00000100:00080000:1.0:1767320686.543288:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320686.543292:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320686.543320:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ea00 x1853163983132544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320686.543358:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983132544:0@lo:38:kworker.0 00000100:00080000:3.0:1767320686.543411:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320686.544762:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983132416:12345-0@lo:400:kworker.0 Request processed in 1680us (1879us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320686.544772:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1144 00000100:00100000:2.0:1767320686.544859:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320686.545022:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983131648 00000100:00100000:0.0:1767320686.545027:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983131648 00000100:00100000:2.0:1767320686.545097:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983132416:0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.545182:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864 00000100:00100000:0.0:1767320686.545189:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983131648:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320686.545355:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983131648:12345-0@lo:400:kworker.0 Request processed in 165us (495us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320686.545362:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864 00010000:00080000:2.0:1767320686.546182:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d180 x1853163983131648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320702 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320686.546199:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983131648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320686.546302:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320686.546388:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132544 00000100:00100000:2.0:1767320686.546394:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132544 00000100:00100000:2.0:1767320686.546428:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2467 00000100:00100000:2.0:1767320686.546485:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983132544:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320686.546505:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320686.546553:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983132544:12345-0@lo:38:kworker.0 Request processed in 118us (251us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320686.546560:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2467 00000100:00080000:1.0:1767320686.546753:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320686.546759:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320686.546762:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983132544:0@lo:38:kworker.0 00000100:00080000:3.0:1767320686.546828:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320686.727409:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320687.749660:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320688.068498:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320688.068517:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983132672:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320688.068597:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320688.068755:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132672 00000100:00100000:1.0:1767320688.068764:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132672 00000100:00100000:1.0:1767320688.068831:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1246 00000100:00100000:1.0:1767320688.068840:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983132672:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320688.068869:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320688.068913:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983132672:12345-0@lo:13:osp-pre-0-1.0 Request processed in 74us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320688.068921:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1246 00000100:00100000:2.0:1767320688.069032:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983132800:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320688.069067:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320688.069069:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320688.069116:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983132672:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320688.069202:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983132800 00000100:00100000:1.0:1767320688.069208:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983132800 00000100:00100000:1.0:1767320688.069248:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247 00000100:00100000:1.0:1767320688.069255:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983132800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320688.069289:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767320688.069357:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320688.071723:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983132800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2460us (2646us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320688.071736:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247 00000100:00100000:2.0:1767320688.075719:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983132800:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320688.772569:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320689.550601:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320689.550643:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915695488 offset 0 length 520: 4 00000100:00100000:3.0:1767320689.550691:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320689.550788:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695744 00000100:00100000:2.0:1767320689.550798:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695744 00000100:00100000:2.0:1767320689.550868:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695616 00000100:00100000:2.0:1767320689.550871:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695616 00000100:00100000:2.0:1767320689.550889:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1865 00000100:00100000:2.0:1767320689.550898:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915695744:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320689.550971:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915695744:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320689.550978:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1865 00000100:00100000:2.0:1767320689.550995:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2468 00000100:00100000:2.0:1767320689.550999:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695616:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320689.551499:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320689.551547:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695616:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 544us (855us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320689.551554:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2468 00000100:00080000:3.0:1767320689.551725:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320689.797477:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320690.820670:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320691.652607:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320666/real 1767320666] req@ffff93463a8e0a80 x1853163983126400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320691 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320691.652652:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463a8e0a80 x1853163983126400/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320691 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320691.652672:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320691.652692:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320691.652696:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983126400:0@lo:250:kworker.0 00000100:00080000:1.0:1767320691.655783:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320691.655795:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320691.655799:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320691.655807:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5914 00000100:00080000:1.0:1767320691.655812:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320691.655819:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320691.655856:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ea00 x1853163983132928/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320691.655922:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320691.655926:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320691.655928:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320691.655932:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5935 00000100:00080000:1.0:1767320691.655934:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320691.655939:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320691.655956:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983132928:0@lo:250:kworker.0 00000100:00100000:1.0:1767320691.655957:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c700 x1853163983133184/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320691.655995:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320691.655998:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320691.656000:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320691.656003:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5935 00000100:00080000:1.0:1767320691.656005:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320691.656012:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000400:00000100:0.0:1767320691.656013:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983132928 offset 0 length 520: 4 00000100:00100000:1.0:1767320691.656024:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d880 x1853163983133440/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320691.656032:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320691.656034:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320691.656036:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320691.656040:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5935 00000100:00080000:1.0:1767320691.656042:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320691.656046:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320691.656061:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163983133568/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320691.656077:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983133184:0@lo:38:kworker.0 00000100:00080000:1.0:1767320691.656086:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320691.656090:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320691.656092:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320691.656096:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5935 00000100:00080000:1.0:1767320691.656099:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320691.656103:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320691.656114:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320691.656118:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934610f51880 x1853163983133952/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320691.656140:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983133440:0@lo:38:kworker.0 00000100:00100000:1.0:1767320691.656157:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983133312:0@lo:400:kworker.0 00000100:00100000:0.0:1767320691.656164:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320691.656182:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983133568:0@lo:38:kworker.0 00000100:00100000:1.0:1767320691.656199:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320691.656214:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320691.656230:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320691.656238:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983133952:0@lo:38:kworker.0 00000100:00100000:1.0:1767320691.656241:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983133696:0@lo:400:kworker.0 00000100:00100000:0.0:1767320691.656266:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320691.656298:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320691.656323:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133184 00000100:00100000:0.0:1767320691.656331:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133184 00000100:00100000:1.0:1767320691.656338:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133440 00000100:00100000:1.0:1767320691.656343:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133440 00000100:00100000:1.0:1767320691.656360:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133312 00000100:00100000:1.0:1767320691.656363:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133312 00000100:00100000:0.0:1767320691.656372:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133952 00000100:00100000:0.0:1767320691.656377:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133952 00000100:00100000:0.0:1767320691.656399:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866 00000100:00100000:1.0:1767320691.656406:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2469 00000100:00100000:0.0:1767320691.656411:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983133312:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320691.656416:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983133184:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320691.656436:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320691.656481:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983133312:12345-0@lo:400:kworker.0 Request processed in 74us (281us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320691.656487:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983133184:12345-0@lo:38:kworker.0 Request processed in 71us (372us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320691.656493:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866 00000100:00100000:1.0:1767320691.656502:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2469 00000100:00100000:0.0:1767320691.656517:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2470 00000100:00100000:0.0:1767320691.656523:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983133440:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320691.656585:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983133824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320691.656586:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133568 00000100:00100000:3.0:1767320691.656593:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133568 00000100:00100000:3.0:1767320691.656619:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 642 00000100:00100000:2.0:1767320691.656634:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320691.656637:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983133568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320691.656650:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320691.656664:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320691.656672:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983133056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320691.656683:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983133568:12345-0@lo:38:kworker.0 Request processed in 47us (469us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320691.656690:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 642 00000100:00100000:2.0:1767320691.656700:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320691.656711:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133056 00000100:00100000:3.0:1767320691.656716:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133056 00010000:00020000:0.0:1767320691.656756:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320691.656781:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983133440:12345-0@lo:38:kworker.0 Request processed in 257us (621us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320691.658392:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2471 00000100:00100000:1.0:1767320691.658404:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983133952:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320691.658424:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320691.658444:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1867 00000100:00100000:3.0:1767320691.658457:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983133056:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320691.658496:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983133952:12345-0@lo:38:kworker.0 Request processed in 91us (2228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320691.658505:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2471 00000100:00100000:3.0:1767320691.658521:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983133056:12345-0@lo:400:kworker.0 Request processed in 68us (1819us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320691.658528:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1867 00000100:00080000:3.0:1767320691.658553:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320691.658557:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133696 00000100:00100000:1.0:1767320691.658565:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133696 00000100:00080000:3.0:1767320691.658595:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320691.658612:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983133824 00000100:00100000:1.0:1767320691.658617:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983133824 00000100:00100000:3.0:1767320691.658620:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1145 00000100:00100000:3.0:1767320691.658638:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983133696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320691.658671:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1146 00000100:00100000:3.0:1767320691.658675:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983133696:12345-0@lo:400:kworker.0 Request processed in 37us (2376us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320691.658684:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983133824:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320691.658688:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1145 00000100:00100000:1.0:1767320691.658731:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983133824:12345-0@lo:400:kworker.0 Request processed in 50us (2107us total) trans 0 rc 0/0 00010000:00080000:3.0:1767320691.658738:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d180 x1853163983133056/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320707 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320691.658743:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1146 00000100:00100000:3.0:1767320691.658757:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983133056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320691.658771:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983133824:0@lo:400:kworker.0 00000100:00080000:1.0:1767320691.658793:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320691.658818:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983133696:0@lo:400:kworker.0 00000100:00080000:1.0:1767320691.658834:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00080000:3.0:1767320691.658836:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396f480 x1853163983133312/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320707 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320691.658842:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320691.658851:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934610f51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983133952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320691.658855:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983133312:0@lo:400:kworker.0 00000100:00080000:1.0:1767320691.658871:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320691.658876:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320691.658880:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983133184:0@lo:38:kworker.0 00000100:00080000:1.0:1767320691.658892:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320691.658896:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320691.658900:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983133440:0@lo:38:kworker.0 00000100:00080000:1.0:1767320691.658909:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320691.658912:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320691.658915:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983133568:0@lo:38:kworker.0 00000100:00100000:0.0:1767320691.659305:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2470 00010000:00010000:3.0:1767320691.844750:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320692.870827:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320693.188943:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320693.188961:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983134080:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320693.189034:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320693.189295:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983134208:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320693.189323:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320693.189410:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134080 00000100:00100000:0.0:1767320693.189419:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134080 00000100:00080000:2.0:1767320693.189687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320693.189724:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134208 00000100:00100000:0.0:1767320693.189728:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134208 00000100:00100000:0.0:1767320693.189974:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248 00000100:00100000:0.0:1767320693.189984:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983134080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320693.190018:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320693.190073:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983134080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (1039us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320693.190080:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248 00000100:00100000:0.0:1767320693.190098:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1249 00000100:00100000:0.0:1767320693.190104:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983134208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320693.190120:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320693.190150:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983134208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (825us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320693.190155:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1249 00000100:00100000:0.0:1767320693.190203:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983134208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320693.190235:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983134080:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320693.190299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320693.892570:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320694.667434:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320694.668082:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696000 00000100:00100000:0.0:1767320694.668094:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696000 00000100:00100000:0.0:1767320694.668265:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1868 00000100:00100000:0.0:1767320694.668314:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915696000:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320694.668401:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915696000:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (977us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320694.668410:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1868 00000100:00100000:1.0:1767320694.668782:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320694.669034:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915695872 00000100:00100000:0.0:1767320694.669040:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915695872 00000100:00100000:0.0:1767320694.669075:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2472 00000100:00100000:0.0:1767320694.669081:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695872:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320694.669102:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320694.669145:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915695872:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 61us (364us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320694.669152:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2472 00000100:00080000:2.0:1767320694.669422:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320694.920321:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320695.940732:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320696.772502:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320696.772514:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320696.772609:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320696.772613:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320696.772617:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320696.772624:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5940 00000100:00080000:0.0:1767320696.772629:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320696.772636:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320696.772658:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163983134464/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320696.772707:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320696.772711:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320696.772713:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320696.772717:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5940 00000100:00080000:0.0:1767320696.772719:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320696.772723:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320696.772736:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163983134720/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320696.772745:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320696.772748:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320696.772749:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320696.772753:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5940 00000100:00080000:0.0:1767320696.772755:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320696.772758:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320696.772770:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580700 x1853163983134848/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320696.772790:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320696.772793:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320696.772795:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320696.772798:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5940 00000100:00080000:0.0:1767320696.772801:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320696.772804:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320696.772813:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163983135232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320696.772840:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983134592:0@lo:400:kworker.0 00000100:00100000:0.0:1767320696.772903:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320696.772940:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320696.772947:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983134976:0@lo:400:kworker.0 00000100:00100000:0.0:1767320696.772975:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320696.773004:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134592 00000100:00100000:0.0:1767320696.773011:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134592 00000100:00100000:0.0:1767320696.773065:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1869 00000100:00100000:0.0:1767320696.773074:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983134592:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320696.773120:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983134592:12345-0@lo:400:kworker.0 Request processed in 47us (216us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320696.773126:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1869 00000100:00080000:0.0:1767320696.773147:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320696.773173:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc700 x1853163983134592/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320712 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320696.773184:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983134592:0@lo:400:kworker.0 00000100:00100000:2.0:1767320696.773268:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983134336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320696.773318:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320696.773349:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320696.773359:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983135104:0@lo:400:kworker.0 00000100:00100000:2.0:1767320696.773381:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320696.773455:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134976 00000100:00100000:1.0:1767320696.773460:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134976 00000100:00100000:1.0:1767320696.773500:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135104 00000100:00100000:1.0:1767320696.773503:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135104 00000100:00100000:1.0:1767320696.773523:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1147 00000100:00100000:1.0:1767320696.773541:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983134976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320696.773582:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983134976:12345-0@lo:400:kworker.0 Request processed in 53us (606us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320696.773588:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1147 00000100:00100000:1.0:1767320696.773601:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1148 00000100:00100000:1.0:1767320696.773606:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983135104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320696.773634:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983135104:12345-0@lo:400:kworker.0 Request processed in 28us (253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320696.773640:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1148 00000100:00100000:1.0:1767320696.773673:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983134976:0@lo:400:kworker.0 00000100:00100000:1.0:1767320696.773691:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134336 00000100:00100000:1.0:1767320696.773693:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134336 00000100:00100000:1.0:1767320696.773716:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870 00000100:00100000:1.0:1767320696.773722:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983134336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320696.773748:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983134336:12345-0@lo:400:kworker.0 Request processed in 26us (429us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320696.773753:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870 00000100:00080000:2.0:1767320696.773813:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320696.773816:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320696.773819:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320696.773835:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983135104:0@lo:400:kworker.0 00010000:00080000:2.0:1767320696.773843:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc380 x1853163983134336/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320712 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320696.773852:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983134336:0@lo:400:kworker.0 00000100:00100000:3.0:1767320696.773933:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983134464:0@lo:38:kworker.0 00000100:00100000:3.0:1767320696.773964:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320696.773990:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983134720:0@lo:38:kworker.0 00000100:00100000:3.0:1767320696.774008:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320696.774031:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983134848:0@lo:38:kworker.0 00000100:00100000:3.0:1767320696.774055:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320696.774072:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983135232:0@lo:38:kworker.0 00000100:00100000:3.0:1767320696.774092:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320696.774123:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134848 00000100:00100000:3.0:1767320696.774128:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134848 00000100:00100000:3.0:1767320696.774148:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 643 00000100:00100000:3.0:1767320696.774153:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983134848:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320696.774169:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320696.774204:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983134848:12345-0@lo:38:kworker.0 Request processed in 49us (147us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320696.774210:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 643 00000100:00100000:3.0:1767320696.774227:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134464 00000100:00100000:3.0:1767320696.774231:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134464 00000100:00100000:3.0:1767320696.774246:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983134720 00000100:00100000:3.0:1767320696.774248:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983134720 00000100:00100000:3.0:1767320696.774256:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135232 00000100:00100000:3.0:1767320696.774258:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135232 00000100:00100000:3.0:1767320696.774267:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2473 00000100:00100000:3.0:1767320696.774298:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983134464:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320696.774308:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320696.774332:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983134464:12345-0@lo:38:kworker.0 Request processed in 34us (367us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320696.774338:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2473 00000100:00100000:3.0:1767320696.774349:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2474 00000100:00100000:3.0:1767320696.774353:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983134720:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320696.774362:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320696.774384:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983134720:12345-0@lo:38:kworker.0 Request processed in 30us (376us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320696.774389:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2474 00000100:00100000:3.0:1767320696.774399:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2475 00000100:00100000:3.0:1767320696.774403:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983135232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320696.774411:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320696.774431:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983135232:12345-0@lo:38:kworker.0 Request processed in 27us (339us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320696.774437:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2475 00000100:00080000:3.0:1767320696.774475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320696.774480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320696.774483:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983134464:0@lo:38:kworker.0 00000100:00080000:3.0:1767320696.774490:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320696.774492:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320696.774495:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983134720:0@lo:38:kworker.0 00000100:00080000:3.0:1767320696.774501:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320696.774504:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320696.774506:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983134848:0@lo:38:kworker.0 00000100:00080000:3.0:1767320696.774512:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320696.774515:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320696.774518:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983135232:0@lo:38:kworker.0 00010000:00010000:1.0:1767320696.964601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320697.993826:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320698.308631:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983135360:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320698.308703:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320698.308807:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135360 00000100:00100000:0.0:1767320698.308816:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135360 00000100:00100000:0.0:1767320698.308892:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1250 00000100:00100000:0.0:1767320698.308902:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983135360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320698.308931:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320698.308988:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983135360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320698.308997:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1250 00000100:00100000:1.0:1767320698.309033:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983135488:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320698.309068:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320698.309207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320698.309244:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983135360:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320698.310427:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135488 00000100:00100000:0.0:1767320698.310438:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135488 00000100:00100000:0.0:1767320698.310518:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251 00000100:00100000:0.0:1767320698.310527:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983135488:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320698.310554:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767320698.310608:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320698.310628:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983135488:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (1557us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320698.310636:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251 00000100:00100000:1.0:1767320698.310681:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983135488:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320699.012542:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320699.788391:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320699.788894:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696128 00000100:00100000:0.0:1767320699.788905:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696128 00000100:00100000:0.0:1767320699.789155:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2476 00000100:00100000:0.0:1767320699.789164:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915696128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320699.789196:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320699.789267:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915696128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 102us (887us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320699.789304:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2476 00000100:00100000:3.0:1767320699.793341:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320699.793527:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696256 00000100:00100000:0.0:1767320699.793539:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696256 00000100:00080000:2.0:1767320699.793705:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320699.793796:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1871 00000100:00100000:0.0:1767320699.793809:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915696256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320699.793904:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915696256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (572us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320699.793913:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1871 00010000:00010000:1.0:1767320700.037322:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320701.063557:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320701.894027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320701.894046:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320701.894105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320701.894110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320701.894114:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320701.894123:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5945 00000100:00080000:0.0:1767320701.894128:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320701.894135:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320701.894162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc380 x1853163983135744/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320701.894209:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320701.894212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320701.894214:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767320701.894219:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983135616:0@lo:400:kworker.0 00000100:00080000:0.0:1767320701.894266:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5945 00000100:00080000:0.0:1767320701.894285:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320701.894289:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320701.894301:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fe680 x1853163983136000/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320701.894310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320701.894313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320701.894315:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320701.894318:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5945 00000100:00080000:0.0:1767320701.894320:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320701.894323:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320701.894334:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983136128/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320701.894357:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320701.894362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320701.894365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320701.894367:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320701.894370:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5945 00000100:00080000:0.0:1767320701.894372:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320701.894375:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320701.894390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911180 x1853163983136512/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320701.894394:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320701.894399:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983136384:0@lo:400:kworker.0 00000100:00100000:0.0:1767320701.894416:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135616 00000100:00100000:3.0:1767320701.894422:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320701.894423:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135616 00000100:00100000:3.0:1767320701.894480:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136384 00000100:00100000:3.0:1767320701.894484:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136384 00000100:00100000:0.0:1767320701.894488:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872 00000100:00100000:0.0:1767320701.894495:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983135616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320701.894507:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1149 00000100:00100000:3.0:1767320701.894514:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983136384:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320701.894560:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983136384:12345-0@lo:400:kworker.0 Request processed in 46us (136us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320701.894567:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1149 00000100:00100000:3.0:1767320701.894604:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983136384:0@lo:400:kworker.0 00000100:00100000:0.0:1767320701.894967:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983135616:12345-0@lo:400:kworker.0 Request processed in 471us (608us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320701.894974:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872 00000100:00100000:2.0:1767320701.895103:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983135872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320701.895134:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320701.895159:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320701.895165:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983136256:0@lo:400:kworker.0 00000100:00100000:2.0:1767320701.895186:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:3.0:1767320701.895196:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163983135616/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320717 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320701.895206:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983135616:0@lo:400:kworker.0 00000100:00080000:2.0:1767320701.895212:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320701.895216:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320701.895225:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136256 00000100:00100000:2.0:1767320701.895228:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136256 00000100:00100000:3.0:1767320701.895235:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983135744:0@lo:38:kworker.0 00000100:00100000:2.0:1767320701.895257:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1150 00000100:00100000:3.0:1767320701.895259:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320701.895263:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983136256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320701.895296:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983136000:0@lo:38:kworker.0 00000100:00100000:2.0:1767320701.895318:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983136256:12345-0@lo:400:kworker.0 Request processed in 55us (131us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320701.895325:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1150 00000100:00100000:3.0:1767320701.895329:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320701.895342:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135872 00000100:00100000:2.0:1767320701.895345:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135872 00000100:00100000:3.0:1767320701.895362:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983136128:0@lo:38:kworker.0 00000100:00100000:2.0:1767320701.895368:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983135744 00000100:00100000:2.0:1767320701.895371:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983135744 00000100:00100000:2.0:1767320701.895379:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136000 00000100:00100000:2.0:1767320701.895381:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136000 00000100:00100000:2.0:1767320701.895390:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1873 00000100:00100000:2.0:1767320701.895396:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983135872:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320701.895397:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320701.895422:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983135872:12345-0@lo:400:kworker.0 Request processed in 26us (289us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320701.895428:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983136512:0@lo:38:kworker.0 00000100:00100000:2.0:1767320701.895445:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1873 00000100:00100000:2.0:1767320701.895457:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2477 00000100:00100000:3.0:1767320701.895457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320701.895461:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983135744:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320701.895478:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136512 00010000:00020000:2.0:1767320701.895479:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320701.895481:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136512 00000100:00100000:3.0:1767320701.895494:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2478 00000100:00100000:3.0:1767320701.895499:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983136000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320701.895509:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320701.895571:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983135744:12345-0@lo:38:kworker.0 Request processed in 107us (310us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320701.895785:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2479 00000100:00100000:0.0:1767320701.895790:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983136512:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320701.895800:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320701.895951:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2477 00000100:00080000:2.0:1767320701.895968:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320701.895971:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320701.895990:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983136256:0@lo:400:kworker.0 00010000:00080000:2.0:1767320701.896002:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fc700 x1853163983135872/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320717 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320701.896012:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983135872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320701.896312:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136128 00000100:00100000:1.0:1767320701.896317:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136128 00000100:00100000:1.0:1767320701.896343:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 644 00000100:00100000:1.0:1767320701.896348:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983136128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320701.896360:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320701.898611:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983136000:12345-0@lo:38:kworker.0 Request processed in 3100us (3270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320701.898629:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2478 00000100:00100000:0.0:1767320701.900798:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983136512:12345-0@lo:38:kworker.0 Request processed in 4996us (5329us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320701.900982:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2479 00000100:00080000:2.0:1767320701.901185:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320701.901191:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320701.901195:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983135744:0@lo:38:kworker.0 00000100:00080000:2.0:1767320701.901215:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320701.901219:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320701.901222:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983136000:0@lo:38:kworker.0 00000100:00100000:1.0:1767320701.902705:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983136128:12345-0@lo:38:kworker.0 Request processed in 6347us (7299us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320701.902718:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 644 00000100:00080000:2.0:1767320701.902822:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320701.902828:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320701.902832:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983136128:0@lo:38:kworker.0 00000100:00080000:2.0:1767320701.902843:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320701.902846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320701.902849:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983136512:0@lo:38:kworker.0 00010000:00010000:1.0:1767320702.086411:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320703.109006:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320703.428894:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983136640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320703.428963:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320703.429249:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983136768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320703.429298:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320703.429394:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136640 00000100:00100000:2.0:1767320703.429402:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136640 00000100:00100000:2.0:1767320703.429470:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136768 00000100:00100000:2.0:1767320703.429473:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136768 00000100:00100000:2.0:1767320703.429491:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1252 00000100:00100000:2.0:1767320703.429498:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983136640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320703.429528:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320703.429576:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983136640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (613us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320703.429583:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1252 00000100:00100000:2.0:1767320703.429601:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253 00000100:00100000:2.0:1767320703.429607:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983136768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320703.429673:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320703.429742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320703.429748:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320703.429795:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983136640:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320703.440022:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983136768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 10386us (10694us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320703.440039:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253 00000100:00100000:3.0:1767320703.441263:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983136768:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320704.137181:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320704.912268:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320704.912448:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320704.926036:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696512 00000100:00100000:1.0:1767320704.926054:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696512 00000100:00100000:1.0:1767320704.926147:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696384 00000100:00100000:1.0:1767320704.926151:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696384 00000100:00100000:1.0:1767320704.926177:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1874 00000100:00100000:1.0:1767320704.926188:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915696512:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320704.926305:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915696512:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 119us (15972us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320704.926324:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1874 00000100:00100000:3.0:1767320704.926622:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2480 00000100:00100000:3.0:1767320704.926630:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915696384:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320704.926651:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320704.926683:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915696384:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (14235us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320704.926690:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2480 00000100:00080000:2.0:1767320704.927724:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320705.156454:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320706.180650:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320707.012454:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320707.012465:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320707.012596:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320707.012601:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320707.012605:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320707.012614:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5950 00000100:00080000:3.0:1767320707.012619:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320707.012627:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320707.012650:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5c00 x1853163983137024/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320707.012824:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983136896:0@lo:400:kworker.0 00000100:00100000:0.0:1767320707.012917:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320707.012990:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320707.012998:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983137152:0@lo:400:kworker.0 00000100:00100000:0.0:1767320707.013023:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320707.013054:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983136896 00000100:00100000:0.0:1767320707.013062:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983136896 00000100:00100000:0.0:1767320707.013107:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137152 00000100:00100000:0.0:1767320707.013110:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137152 00000100:00100000:0.0:1767320707.013151:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1875 00000100:00100000:0.0:1767320707.013159:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983136896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320707.013209:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983136896:12345-0@lo:400:kworker.0 Request processed in 49us (291us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320707.013216:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1875 00000100:00100000:0.0:1767320707.013234:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876 00000100:00100000:0.0:1767320707.013240:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983137152:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320707.013295:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983137152:12345-0@lo:400:kworker.0 Request processed in 32us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320707.013301:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876 00010000:00080000:0.0:1767320707.013338:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7b80 x1853163983137152/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320723 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320707.013363:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983137152:0@lo:400:kworker.0 00010000:00080000:0.0:1767320707.013390:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4e00 x1853163983136896/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320723 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320707.013398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983136896:0@lo:400:kworker.0 00000100:00080000:2.0:1767320707.013493:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320707.013548:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320707.013553:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320707.013555:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320707.013560:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5950 00000100:00080000:3.0:1767320707.013563:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320707.013567:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320707.013585:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163983137280/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320707.013596:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320707.013599:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320707.013601:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320707.013604:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5950 00000100:00080000:3.0:1767320707.013606:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320707.013609:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320707.013619:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7b80 x1853163983137408/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320707.013675:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767320707.013856:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320707.013861:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320707.013862:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320707.013866:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5950 00000100:00080000:3.0:1767320707.013869:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320707.013873:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320707.013886:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163983137792/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320707.013907:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320707.013915:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983137664:0@lo:400:kworker.0 00000100:00100000:3.0:1767320707.013952:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.014053:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983137024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320707.014085:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.014122:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983137280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320707.014144:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.014161:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983137408:0@lo:38:kworker.0 00000100:00100000:1.0:1767320707.014183:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.014204:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983137792:0@lo:38:kworker.0 00000100:00100000:1.0:1767320707.014224:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.014255:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137664 00000100:00100000:1.0:1767320707.014260:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137664 00000100:00100000:1.0:1767320707.014322:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1151 00000100:00100000:1.0:1767320707.014329:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983137664:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320707.014384:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983137664:12345-0@lo:400:kworker.0 Request processed in 55us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320707.014391:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1151 00000100:00100000:1.0:1767320707.014440:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137408 00000100:00100000:1.0:1767320707.014444:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137408 00000100:00100000:1.0:1767320707.014465:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 645 00000100:00100000:1.0:1767320707.014469:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983137408:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320707.014486:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320707.014519:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983137408:12345-0@lo:38:kworker.0 Request processed in 48us (334us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320707.014524:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 645 00000100:00100000:1.0:1767320707.014544:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137024 00000100:00100000:1.0:1767320707.014547:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137024 00000100:00100000:1.0:1767320707.014561:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137280 00000100:00100000:1.0:1767320707.014563:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137280 00000100:00100000:1.0:1767320707.014576:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137792 00000100:00100000:1.0:1767320707.014579:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137792 00000100:00100000:1.0:1767320707.014588:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2481 00000100:00100000:1.0:1767320707.014593:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320707.014602:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320707.014627:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137024:12345-0@lo:38:kworker.0 Request processed in 34us (541us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320707.014633:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2481 00000100:00100000:1.0:1767320707.014643:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2482 00000100:00100000:1.0:1767320707.014646:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320707.014654:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320707.014674:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137280:12345-0@lo:38:kworker.0 Request processed in 26us (529us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320707.014678:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2482 00000100:00100000:1.0:1767320707.014686:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2483 00000100:00100000:1.0:1767320707.014689:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320707.014708:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320707.014727:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983137792:12345-0@lo:38:kworker.0 Request processed in 36us (503us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320707.014732:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2483 00000100:00080000:1.0:1767320707.014758:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320707.014762:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320707.014766:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983137024:0@lo:38:kworker.0 00000100:00080000:1.0:1767320707.014773:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320707.014776:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320707.014778:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983137280:0@lo:38:kworker.0 00000100:00080000:1.0:1767320707.014784:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320707.014787:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320707.014789:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983137408:0@lo:38:kworker.0 00000100:00080000:1.0:1767320707.014796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320707.014799:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320707.014802:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983137792:0@lo:38:kworker.0 00000100:00100000:1.0:1767320707.014847:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983137664:0@lo:400:kworker.0 00000100:00080000:1.0:1767320707.014879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320707.016865:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983137536:0@lo:400:kworker.0 00000100:00100000:2.0:1767320707.016935:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320707.017048:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137536 00000100:00100000:1.0:1767320707.017058:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137536 00000100:00100000:1.0:1767320707.017166:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1152 00000100:00100000:1.0:1767320707.017175:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983137536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320707.017237:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983137536:12345-0@lo:400:kworker.0 Request processed in 64us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320707.017244:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1152 00000100:00100000:0.0:1767320707.017335:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983137536:0@lo:400:kworker.0 00000100:00080000:2.0:1767320707.021594:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767320707.209419:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320708.229162:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320708.548536:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320708.548561:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983137920:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320708.548638:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320708.548770:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983137920 00000100:00100000:2.0:1767320708.548778:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983137920 00000100:00100000:2.0:1767320708.548829:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1254 00000100:00100000:2.0:1767320708.548837:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983137920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320708.548865:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320708.548913:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983137920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320708.548920:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1254 00000100:00100000:0.0:1767320708.549078:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983137920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320708.552006:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320708.552027:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983138048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320708.552092:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320708.552400:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138048 00000100:00100000:0.0:1767320708.552409:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138048 00000100:00100000:0.0:1767320708.552461:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1255 00000100:00100000:0.0:1767320708.552469:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983138048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320708.552495:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320708.552552:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983138048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (459us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320708.552559:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1255 00000100:00100000:1.0:1767320708.552645:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983138048:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320709.256933:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320710.035424:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320710.035479:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320710.035596:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696768 00000100:00100000:1.0:1767320710.035607:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696768 00000100:00100000:1.0:1767320710.035688:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915696640 00000100:00100000:1.0:1767320710.035692:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915696640 00000100:00100000:1.0:1767320710.035712:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1877 00000100:00100000:1.0:1767320710.035722:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915696768:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320710.035793:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915696768:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320710.035801:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1877 00000100:00100000:1.0:1767320710.035822:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2484 00000100:00100000:1.0:1767320710.035827:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915696640:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320710.035848:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320710.035879:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915696640:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 49us (399us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320710.035886:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2484 00000100:00080000:3.0:1767320710.036233:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320710.276536:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320711.300995:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320712.132709:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320712.132718:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320712.132815:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320712.132819:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320712.132823:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320712.132830:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5955 00000100:00080000:3.0:1767320712.132835:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320712.132841:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320712.132860:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8a80 x1853163983138304/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320712.132914:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320712.132917:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320712.132919:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1767320712.132923:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983138176:0@lo:400:kworker.0 00000100:00080000:3.0:1767320712.132924:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5955 00000100:00080000:3.0:1767320712.132929:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320712.132935:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320712.132946:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8700 x1853163983138560/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320712.132955:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320712.132958:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320712.132960:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320712.132963:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5955 00000100:00080000:3.0:1767320712.132966:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320712.132969:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320712.132979:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163983138688/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320712.132997:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320712.133004:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320712.133008:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320712.133009:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320712.133022:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5955 00000100:00080000:3.0:1767320712.133025:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320712.133028:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320712.133039:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ca80 x1853163983139072/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320712.133062:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983138816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320712.133067:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320712.133075:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983138944:0@lo:400:kworker.0 00000100:00100000:3.0:1767320712.133101:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320712.133141:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320712.133144:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138816 00000100:00100000:3.0:1767320712.133148:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983138432:0@lo:400:kworker.0 00000100:00100000:1.0:1767320712.133154:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138816 00000100:00100000:3.0:1767320712.133172:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320712.133212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138176 00000100:00100000:3.0:1767320712.133217:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138176 00000100:00100000:1.0:1767320712.133239:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1153 00000100:00100000:3.0:1767320712.133259:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138432 00000100:00100000:3.0:1767320712.133263:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138432 00000100:00100000:1.0:1767320712.133264:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983138816:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320712.133306:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878 00000100:00100000:3.0:1767320712.133314:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983138176:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320712.133352:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983138816:12345-0@lo:400:kworker.0 Request processed in 102us (249us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320712.133356:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983138176:12345-0@lo:400:kworker.0 Request processed in 41us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320712.133365:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1153 00000100:00100000:3.0:1767320712.133368:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878 00000100:00100000:3.0:1767320712.133388:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1879 00000100:00100000:3.0:1767320712.133395:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983138432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320712.133419:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983138432:12345-0@lo:400:kworker.0 Request processed in 25us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320712.133422:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983138816:0@lo:400:kworker.0 00000100:00100000:3.0:1767320712.133430:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1879 00000100:00080000:3.0:1767320712.133449:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320712.133453:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9880 x1853163983138432/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320728 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320712.133455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320712.133466:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983138432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320712.133481:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320712.133497:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138944 00000100:00100000:1.0:1767320712.133502:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138944 00010000:00080000:3.0:1767320712.133509:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9500 x1853163983138176/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320728 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320712.133520:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983138176:0@lo:400:kworker.0 00000100:00100000:1.0:1767320712.133537:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1154 00000100:00080000:3.0:1767320712.133542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320712.133547:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983138944:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320712.133591:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983138944:12345-0@lo:400:kworker.0 Request processed in 46us (110us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320712.133597:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1154 00000100:00100000:0.0:1767320712.133707:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983138304:0@lo:38:kworker.0 00000100:00100000:0.0:1767320712.133751:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320712.133786:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983138560:0@lo:38:kworker.0 00000100:00100000:0.0:1767320712.133806:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320712.133831:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983138688:0@lo:38:kworker.0 00000100:00100000:0.0:1767320712.133855:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320712.133877:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983139072:0@lo:38:kworker.0 00000100:00100000:0.0:1767320712.133897:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320712.133925:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138688 00000100:00100000:0.0:1767320712.133929:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138688 00000100:00100000:0.0:1767320712.133951:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 646 00000100:00100000:0.0:1767320712.133956:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983138688:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320712.133987:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320712.134019:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983138688:12345-0@lo:38:kworker.0 Request processed in 61us (163us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320712.134026:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 646 00000100:00100000:0.0:1767320712.134046:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138304 00000100:00100000:0.0:1767320712.134049:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138304 00000100:00100000:0.0:1767320712.134066:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983138560 00000100:00100000:0.0:1767320712.134069:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983138560 00000100:00100000:0.0:1767320712.134079:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139072 00000100:00100000:0.0:1767320712.134081:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139072 00000100:00100000:0.0:1767320712.134089:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2485 00000100:00100000:0.0:1767320712.134093:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983138304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320712.134103:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320712.134127:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983138304:12345-0@lo:38:kworker.0 Request processed in 32us (376us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320712.134133:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2485 00000100:00100000:0.0:1767320712.134143:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2486 00000100:00100000:0.0:1767320712.134146:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983138560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320712.134155:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320712.134177:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983138560:12345-0@lo:38:kworker.0 Request processed in 29us (370us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320712.134181:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2486 00000100:00100000:0.0:1767320712.134191:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2487 00000100:00100000:0.0:1767320712.134194:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983139072:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320712.134202:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320712.134220:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983139072:12345-0@lo:38:kworker.0 Request processed in 24us (321us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320712.134225:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2487 00000100:00080000:0.0:1767320712.134262:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320712.134267:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320712.134300:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983138304:0@lo:38:kworker.0 00000100:00080000:0.0:1767320712.134312:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320712.134316:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320712.134319:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983138560:0@lo:38:kworker.0 00000100:00080000:0.0:1767320712.134327:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320712.134330:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320712.134332:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983138688:0@lo:38:kworker.0 00000100:00080000:0.0:1767320712.134339:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320712.134342:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320712.134345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983139072:0@lo:38:kworker.0 00000100:00100000:3.0:1767320712.134747:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983138944:0@lo:400:kworker.0 00010000:00010000:1.0:1767320712.325186:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320713.349146:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320713.668924:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983139200:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320713.669003:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320713.669148:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983139328:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320713.669178:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320713.675984:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139200 00000100:00100000:3.0:1767320713.676002:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139200 00000100:00100000:3.0:1767320713.676088:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139328 00000100:00100000:3.0:1767320713.676093:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139328 00000100:00100000:3.0:1767320713.676119:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256 00000100:00100000:3.0:1767320713.676129:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983139200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320713.676162:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320713.676303:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983139200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 175us (7298us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320713.676310:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256 00000100:00100000:3.0:1767320713.676328:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257 00000100:00100000:3.0:1767320713.676335:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983139328:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320713.676351:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320713.676399:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983139328:12345-0@lo:13:osp-pre-0-1.0 Request processed in 65us (7220us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320713.676405:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257 00000100:00100000:2.0:1767320713.676701:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983139200:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320713.677325:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983139328:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320714.124198:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320714.124243:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915696896 offset 0 length 520: 4 00000100:00100000:3.0:1767320714.124264:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320714.124408:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697152 00000100:00100000:1.0:1767320714.124417:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697152 00000100:00100000:1.0:1767320714.124479:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697024 00000100:00100000:1.0:1767320714.124483:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697024 00000100:00100000:1.0:1767320714.124502:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1880 00000100:00100000:1.0:1767320714.124512:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915697152:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320714.124585:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915697152:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (396us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320714.124593:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1880 00000100:00100000:1.0:1767320714.124609:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2488 00000100:00100000:1.0:1767320714.124613:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915697024:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320714.124796:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320714.124826:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915697024:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 211us (560us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320714.124833:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2488 00000100:00080000:3.0:1767320714.125858:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320714.380860:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320715.396648:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320716.228422:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320691/real 1767320691] req@ffff93463396ea00 x1853163983132928/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320716 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320716.228458:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463396ea00 x1853163983132928/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320716 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320716.228479:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320716.228508:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320716.228513:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983132928:0@lo:250:kworker.0 00000100:00080000:0.0:1767320716.228601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320716.228606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320716.228609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320716.228614:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5940 00000100:00080000:0.0:1767320716.228617:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320716.228661:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320716.228707:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911880 x1853163983139456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320716.228770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320716.228774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320716.228776:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320716.228779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5960 00000100:00080000:0.0:1767320716.228782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320716.228785:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320716.228799:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398ca80 x1853163983139712/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320716.228827:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320716.228843:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983139584:0@lo:400:kworker.0 00000100:00080000:0.0:1767320716.228857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320716.228862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320716.228864:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320716.228869:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5960 00000100:00080000:0.0:1767320716.228871:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320716.228875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320716.228889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55500 x1853163983139968/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320716.228898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320716.228901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320716.228903:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320716.228906:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5960 00000100:00080000:0.0:1767320716.228908:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320716.228912:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320716.228923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54700 x1853163983140096/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320716.228932:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320716.228970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320716.228974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320716.228976:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320716.228980:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5960 00000100:00080000:0.0:1767320716.228983:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320716.228987:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320716.229000:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55f80 x1853163983140480/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320716.229014:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983140352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320716.229028:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983139840:0@lo:400:kworker.0 00000100:00100000:3.0:1767320716.229063:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320716.229072:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320716.229101:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320716.229107:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983140224:0@lo:400:kworker.0 00000100:00100000:0.0:1767320716.229130:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320716.229162:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139584 00000100:00100000:0.0:1767320716.229180:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139584 00000100:00100000:3.0:1767320716.229224:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139840 00000100:00100000:3.0:1767320716.229230:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139840 00000100:00100000:0.0:1767320716.229237:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1881 00000100:00100000:0.0:1767320716.229250:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983139584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320716.229297:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882 00000100:00100000:3.0:1767320716.229306:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983139840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320716.229351:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983139584:12345-0@lo:400:kworker.0 Request processed in 104us (422us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320716.229361:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1881 00000100:00100000:3.0:1767320716.229375:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983139840:12345-0@lo:400:kworker.0 Request processed in 56us (288us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320716.229385:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882 00000100:00080000:3.0:1767320716.229410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320716.229444:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57480 x1853163983139840/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320732 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:3.0:1767320716.229445:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f100 x1853163983139584/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320732 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320716.229474:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983139584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320716.229475:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983139840:0@lo:400:kworker.0 00000100:00100000:2.0:1767320716.229768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983139456:0@lo:250:kworker.0 00000400:00000100:2.0:1767320716.229797:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983139456 offset 0 length 520: 4 00000100:00100000:2.0:1767320716.229818:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983139712:0@lo:38:kworker.0 00000100:00100000:2.0:1767320716.229842:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320716.229870:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983139968:0@lo:38:kworker.0 00000100:00100000:2.0:1767320716.229892:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320716.229916:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983140096:0@lo:38:kworker.0 00000100:00100000:2.0:1767320716.229972:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320716.229992:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983140480:0@lo:38:kworker.0 00000100:00100000:2.0:1767320716.230013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320716.230043:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140096 00000100:00100000:2.0:1767320716.230048:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140096 00000100:00100000:2.0:1767320716.230074:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 647 00000100:00100000:2.0:1767320716.230079:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983140096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320716.230097:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320716.230133:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983140096:12345-0@lo:38:kworker.0 Request processed in 52us (160us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320716.230140:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 647 00000100:00100000:2.0:1767320716.230164:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139712 00000100:00100000:2.0:1767320716.230168:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139712 00000100:00100000:2.0:1767320716.230201:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983139968 00000100:00100000:2.0:1767320716.230205:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983139968 00000100:00100000:2.0:1767320716.230213:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140480 00000100:00100000:2.0:1767320716.230216:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140480 00000100:00100000:2.0:1767320716.230226:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2489 00000100:00100000:2.0:1767320716.230230:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983139712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320716.230240:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320716.230266:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983139712:12345-0@lo:38:kworker.0 Request processed in 35us (423us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320716.230295:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2489 00000100:00100000:2.0:1767320716.230309:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2490 00000100:00100000:2.0:1767320716.230312:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983139968:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320716.230321:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320716.230342:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983139968:12345-0@lo:38:kworker.0 Request processed in 28us (450us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320716.230347:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2490 00000100:00100000:2.0:1767320716.230358:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2491 00000100:00100000:2.0:1767320716.230361:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983140480:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320716.230368:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320716.230388:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983140480:12345-0@lo:38:kworker.0 Request processed in 25us (374us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320716.230393:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2491 00000100:00080000:2.0:1767320716.230423:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320716.230428:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320716.230431:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983139712:0@lo:38:kworker.0 00000100:00080000:2.0:1767320716.230439:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320716.230443:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320716.230445:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983139968:0@lo:38:kworker.0 00000100:00080000:2.0:1767320716.230454:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320716.230457:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320716.230460:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983140096:0@lo:38:kworker.0 00000100:00080000:2.0:1767320716.230466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320716.230469:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320716.230472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983140480:0@lo:38:kworker.0 00000100:00100000:1.0:1767320716.230621:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140352 00000100:00100000:1.0:1767320716.230627:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140352 00000100:00100000:1.0:1767320716.230685:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140224 00000100:00100000:1.0:1767320716.230689:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140224 00000100:00100000:1.0:1767320716.230711:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1155 00000100:00100000:1.0:1767320716.230718:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983140352:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320716.230764:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983140352:12345-0@lo:400:kworker.0 Request processed in 47us (1702us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320716.230771:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1155 00000100:00100000:1.0:1767320716.230783:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1156 00000100:00100000:1.0:1767320716.230789:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983140224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320716.230824:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983140224:12345-0@lo:400:kworker.0 Request processed in 35us (1693us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320716.230831:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1156 00000100:00100000:1.0:1767320716.230870:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983140224:0@lo:400:kworker.0 00000100:00100000:3.0:1767320716.230990:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983140352:0@lo:400:kworker.0 00000100:00080000:2.0:1767320716.231064:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320716.231068:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:3.0:1767320716.420957:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320717.444605:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320718.468730:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320718.788729:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983140736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320718.788765:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983140608:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320718.788805:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320718.788846:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320718.788922:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140736 00000100:00100000:0.0:1767320718.788931:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140736 00000100:00100000:2.0:1767320718.788937:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140608 00000100:00100000:2.0:1767320718.788945:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140608 00000100:00100000:1.0:1767320718.789028:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1258 00000100:00100000:1.0:1767320718.789039:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983140736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320718.789069:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320718.789116:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983140736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (310us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320718.789134:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1258 00000100:00100000:1.0:1767320718.789195:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983140736:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320718.798224:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320718.798236:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320718.799537:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259 00000100:00100000:2.0:1767320718.799558:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983140608:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320718.799591:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320718.799676:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983140608:12345-0@lo:13:osp-pre-0-1.0 Request processed in 130us (10829us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320718.799683:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259 00000100:00100000:3.0:1767320718.799831:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983140608:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320719.243475:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320719.243841:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697408 00000100:00100000:0.0:1767320719.243853:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697408 00000100:00100000:0.0:1767320719.243938:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1883 00000100:00100000:0.0:1767320719.243948:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915697408:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320719.244029:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915697408:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (564us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320719.244037:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1883 00000100:00080000:0.0:1767320719.244403:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320719.253237:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320719.253367:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697280 00000100:00100000:3.0:1767320719.253378:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697280 00000100:00100000:3.0:1767320719.253485:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2492 00000100:00100000:3.0:1767320719.253505:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915697280:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320719.253530:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320719.253601:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915697280:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 94us (373us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320719.253608:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2492 00010000:00010000:3.0:1767320719.492605:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320720.517319:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320721.348705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320721.348716:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320721.348804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320721.348808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320721.348812:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320721.348820:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5964 00000100:00080000:3.0:1767320721.348825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320721.348832:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320721.348854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190a300 x1853163983140992/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320721.348900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320721.348905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320721.348907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320721.348911:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5964 00000100:00080000:3.0:1767320721.348913:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320721.348917:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320721.348930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909500 x1853163983141248/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320721.348938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320721.348941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320721.348943:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320721.348947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5964 00000100:00080000:3.0:1767320721.348949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320721.348952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320721.348963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631908700 x1853163983141376/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320721.348984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320721.348987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320721.348989:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320721.348992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5964 00000100:00080000:3.0:1767320721.348995:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320721.348997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320721.349007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909f80 x1853163983141760/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320721.349033:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983141120:0@lo:400:kworker.0 00000100:00100000:3.0:1767320721.349107:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320721.349152:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320721.349159:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983141504:0@lo:400:kworker.0 00000100:00100000:3.0:1767320721.349186:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320721.349219:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141120 00000100:00100000:3.0:1767320721.349227:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141120 00000100:00100000:3.0:1767320721.349343:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884 00000100:00100000:3.0:1767320721.349353:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983141120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320721.349405:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983141120:12345-0@lo:400:kworker.0 Request processed in 53us (297us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320721.349412:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884 00000100:00080000:3.0:1767320721.349433:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320721.349459:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190b480 x1853163983141120/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320737 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320721.349472:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983141120:0@lo:400:kworker.0 00000100:00100000:2.0:1767320721.349566:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983140992:0@lo:38:kworker.0 00000100:00100000:2.0:1767320721.349599:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320721.349624:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983141248:0@lo:38:kworker.0 00000100:00100000:2.0:1767320721.349658:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320721.349675:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983141376:0@lo:38:kworker.0 00000100:00100000:2.0:1767320721.349701:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320721.349718:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983141760:0@lo:38:kworker.0 00000100:00100000:2.0:1767320721.349736:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320721.349764:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141376 00000100:00100000:2.0:1767320721.349774:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141376 00000100:00100000:2.0:1767320721.349797:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 648 00000100:00100000:2.0:1767320721.349802:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983141376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320721.349820:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320721.349852:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983141376:12345-0@lo:38:kworker.0 Request processed in 49us (151us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320721.349859:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 648 00000100:00100000:2.0:1767320721.349877:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140992 00000100:00100000:2.0:1767320721.349881:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140992 00000100:00100000:2.0:1767320721.349899:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141248 00000100:00100000:2.0:1767320721.349902:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141248 00000100:00100000:2.0:1767320721.349909:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141760 00000100:00100000:2.0:1767320721.349911:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141760 00000100:00100000:2.0:1767320721.349920:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2493 00000100:00100000:2.0:1767320721.349925:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983140992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320721.349934:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320721.349959:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983140992:12345-0@lo:38:kworker.0 Request processed in 34us (360us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320721.349965:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2493 00000100:00100000:2.0:1767320721.349975:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2494 00000100:00100000:2.0:1767320721.349979:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983141248:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320721.349987:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320721.350007:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983141248:12345-0@lo:38:kworker.0 Request processed in 27us (349us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320721.350013:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2494 00000100:00100000:2.0:1767320721.350023:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2495 00000100:00100000:2.0:1767320721.350027:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983141760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320721.350035:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320721.350056:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983141760:12345-0@lo:38:kworker.0 Request processed in 27us (318us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320721.350061:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2495 00000100:00080000:2.0:1767320721.350104:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320721.350109:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320721.350113:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983140992:0@lo:38:kworker.0 00000100:00080000:2.0:1767320721.350120:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320721.350123:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320721.350125:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983141248:0@lo:38:kworker.0 00000100:00080000:2.0:1767320721.350132:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320721.350135:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320721.350144:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983141376:0@lo:38:kworker.0 00000100:00080000:2.0:1767320721.350153:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320721.350155:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320721.350158:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983141760:0@lo:38:kworker.0 00000100:00100000:1.0:1767320721.350511:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983141632:0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350566:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320721.350600:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320721.350606:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983140864:0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350627:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320721.350663:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141504 00000100:00100000:1.0:1767320721.350668:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141504 00000100:00100000:1.0:1767320721.350705:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141632 00000100:00100000:1.0:1767320721.350709:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141632 00000100:00100000:1.0:1767320721.350729:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1157 00000100:00100000:1.0:1767320721.350735:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983141504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350775:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983141504:12345-0@lo:400:kworker.0 Request processed in 40us (1588us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320721.350781:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1157 00000100:00100000:1.0:1767320721.350794:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1158 00000100:00100000:1.0:1767320721.350800:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983141632:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350827:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983141632:12345-0@lo:400:kworker.0 Request processed in 28us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320721.350832:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1158 00000100:00100000:1.0:1767320721.350847:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983140864 00000100:00100000:1.0:1767320721.350850:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983140864 00000100:00100000:1.0:1767320721.350872:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1885 00000100:00100000:1.0:1767320721.350878:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983140864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350903:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983140864:12345-0@lo:400:kworker.0 Request processed in 26us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320721.350908:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1885 00010000:00080000:1.0:1767320721.350941:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631909880 x1853163983140864/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320737 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320721.350951:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983140864:0@lo:400:kworker.0 00000100:00100000:1.0:1767320721.350961:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983141632:0@lo:400:kworker.0 00000100:00080000:3.0:1767320721.351101:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320721.351107:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320721.351117:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320721.351142:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983141504:0@lo:400:kworker.0 00010000:00010000:1.0:1767320721.540600:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320722.564671:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320723.589085:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320723.909852:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983142016:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320723.909974:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320723.910013:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983141888:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320723.910057:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320723.910178:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142016 00000100:00100000:0.0:1767320723.910186:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142016 00000100:00100000:0.0:1767320723.910298:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983141888 00000100:00100000:0.0:1767320723.910302:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983141888 00000100:00100000:0.0:1767320723.910322:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260 00000100:00100000:0.0:1767320723.910331:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983142016:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320723.910332:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320723.910336:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:0.0:1767320723.910381:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320723.910430:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983142016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (456us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320723.910437:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260 00000100:00100000:0.0:1767320723.910452:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1261 00000100:00100000:0.0:1767320723.910459:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983141888:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320723.910473:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320723.910500:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983141888:12345-0@lo:13:osp-pre-0-1.0 Request processed in 42us (442us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320723.910506:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1261 00000100:00100000:0.0:1767320723.910541:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983141888:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320723.910579:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983142016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320724.364101:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320724.364149:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320724.364193:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697664 00000100:00100000:0.0:1767320724.364203:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697664 00000100:00080000:3.0:1767320724.364245:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320724.364312:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697536 00000100:00100000:0.0:1767320724.364316:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697536 00000100:00100000:0.0:1767320724.364345:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1886 00000100:00100000:0.0:1767320724.364357:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915697664:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320724.364443:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915697664:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (352us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320724.364450:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2496 00000100:00100000:0.0:1767320724.364454:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1886 00000100:00100000:3.0:1767320724.364456:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915697536:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320724.364474:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320724.364501:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915697536:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320724.364507:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2496 00010000:00010000:1.0:1767320724.612518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320725.636542:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320726.468451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320726.468462:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320726.468531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320726.468535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320726.468539:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320726.468547:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5969 00000100:00080000:2.0:1767320726.468553:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320726.468561:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320726.468639:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582300 x1853163983142272/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320726.468670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320726.468685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320726.468687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320726.468691:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5969 00000100:00080000:2.0:1767320726.468693:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320726.468698:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320726.468711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581500 x1853163983142528/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320726.468721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320726.468725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320726.468726:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320726.468730:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5969 00000100:00080000:2.0:1767320726.468733:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320726.468736:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320726.468747:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580700 x1853163983142656/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320726.468769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320726.468773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320726.468776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320726.468779:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5969 00000100:00080000:2.0:1767320726.468781:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320726.468784:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320726.468794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581f80 x1853163983143040/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320726.468861:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983142400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320726.468872:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983142912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320726.468967:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983142272:0@lo:38:kworker.0 00000100:00100000:1.0:1767320726.469266:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320726.469323:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320726.469330:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983142144:0@lo:400:kworker.0 00000100:00100000:1.0:1767320726.469356:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320726.471911:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142912 00000100:00100000:2.0:1767320726.471927:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142912 00000100:00100000:2.0:1767320726.472020:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1159 00000100:00100000:2.0:1767320726.472030:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983142912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320726.472125:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983142912:12345-0@lo:400:kworker.0 Request processed in 97us (2860us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320726.472133:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1159 00000100:00100000:2.0:1767320726.472160:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142144 00000100:00100000:2.0:1767320726.472164:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142144 00000100:00100000:2.0:1767320726.472191:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1887 00000100:00100000:2.0:1767320726.472197:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983142144:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320726.472228:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983142144:12345-0@lo:400:kworker.0 Request processed in 32us (2872us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320726.472234:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1887 00010000:00080000:2.0:1767320726.472308:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581880 x1853163983142144/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320742 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320726.472323:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983142144:0@lo:400:kworker.0 00000100:00100000:2.0:1767320726.472337:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983142912:0@lo:400:kworker.0 00000100:00080000:2.0:1767320726.472384:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320726.472389:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320726.473134:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320726.473352:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142400 00000100:00100000:2.0:1767320726.473359:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142400 00000100:00100000:2.0:1767320726.473401:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888 00000100:00100000:2.0:1767320726.473409:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983142400:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320726.473459:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983142400:12345-0@lo:400:kworker.0 Request processed in 51us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320726.473467:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888 00000100:00080000:3.0:1767320726.473636:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320726.473663:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583480 x1853163983142400/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320742 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320726.473686:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983142400:0@lo:400:kworker.0 00000100:00100000:0.0:1767320726.474104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320726.474143:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983142528:0@lo:38:kworker.0 00000100:00100000:0.0:1767320726.474170:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320726.474292:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983142656:0@lo:38:kworker.0 00000100:00100000:0.0:1767320726.474319:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320726.474348:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983143040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320726.474372:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320726.474705:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983142784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320726.474738:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320726.475418:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142784 00000100:00100000:1.0:1767320726.475433:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142784 00000100:00100000:1.0:1767320726.475484:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1160 00000100:00100000:1.0:1767320726.475492:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983142784:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320726.475537:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983142784:12345-0@lo:400:kworker.0 Request processed in 46us (798us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320726.475545:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1160 00000100:00080000:1.0:1767320726.475565:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320726.475603:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983142784:0@lo:400:kworker.0 00000100:00100000:1.0:1767320726.475647:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142656 00000100:00100000:1.0:1767320726.475650:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142656 00000100:00100000:1.0:1767320726.475668:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 649 00000100:00100000:1.0:1767320726.475673:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983142656:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320726.475690:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320726.475719:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983142656:12345-0@lo:38:kworker.0 Request processed in 45us (1399us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320726.475725:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 649 00000100:00080000:0.0:1767320726.481047:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320726.481062:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320726.481068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983142656:0@lo:38:kworker.0 00000100:00100000:3.0:1767320726.487937:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142272 00000100:00100000:3.0:1767320726.487955:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142272 00000100:00100000:3.0:1767320726.488051:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983142528 00000100:00100000:3.0:1767320726.488055:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983142528 00000100:00100000:3.0:1767320726.488077:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143040 00000100:00100000:3.0:1767320726.488080:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143040 00000100:00100000:3.0:1767320726.488097:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2497 00000100:00100000:3.0:1767320726.488105:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983142272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320726.488131:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320726.488220:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983142272:12345-0@lo:38:kworker.0 Request processed in 113us (14115us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320726.488367:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2497 00000100:00100000:3.0:1767320726.488386:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2498 00000100:00100000:3.0:1767320726.488391:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983142528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320726.488401:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320726.488429:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983142528:12345-0@lo:38:kworker.0 Request processed in 37us (14258us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320726.488434:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2498 00000100:00100000:3.0:1767320726.488446:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2499 00000100:00100000:3.0:1767320726.488449:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983143040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320726.488458:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320726.488481:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983143040:12345-0@lo:38:kworker.0 Request processed in 30us (14108us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320726.488487:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2499 00000100:00080000:0.0:1767320726.488867:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320726.488874:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320726.488878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983142272:0@lo:38:kworker.0 00000100:00080000:0.0:1767320726.488890:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320726.488893:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320726.488896:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983142528:0@lo:38:kworker.0 00000100:00080000:0.0:1767320726.488904:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320726.488907:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320726.488909:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983143040:0@lo:38:kworker.0 00010000:00010000:1.0:1767320726.660560:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320727.688108:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320728.710752:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320729.028586:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320729.028611:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983143168:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320729.028763:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320729.029309:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320729.029321:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983143296:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320729.029370:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320729.029507:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143168 00000100:00100000:0.0:1767320729.029516:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143168 00000100:00100000:0.0:1767320729.029595:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143296 00000100:00100000:0.0:1767320729.029599:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143296 00000100:00100000:0.0:1767320729.029640:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1262 00000100:00100000:0.0:1767320729.029648:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983143168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320729.029679:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320729.029728:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983143168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (964us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320729.029735:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1262 00000100:00080000:3.0:1767320729.029954:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320729.029958:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320729.029996:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983143168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320729.037355:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263 00000100:00100000:2.0:1767320729.037521:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983143296:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320729.037600:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320729.037767:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983143296:12345-0@lo:13:osp-pre-0-1.0 Request processed in 399us (8396us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320729.037776:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263 00000100:00100000:3.0:1767320729.038205:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983143296:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320729.484585:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320729.484664:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320729.484742:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697920 00000100:00100000:2.0:1767320729.484753:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697920 00000100:00100000:2.0:1767320729.484844:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915697792 00000100:00100000:2.0:1767320729.484848:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915697792 00000100:00100000:2.0:1767320729.484887:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1889 00000100:00100000:2.0:1767320729.484900:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915697920:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320729.484999:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915697920:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (425us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320729.485008:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1889 00000100:00100000:2.0:1767320729.485033:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2500 00000100:00100000:2.0:1767320729.485039:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915697792:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320729.485060:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320729.485087:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915697792:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (423us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320729.485094:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2500 00000100:00080000:1.0:1767320729.485164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320729.732616:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320730.760223:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320731.588489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320731.588500:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320731.588577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320731.588582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320731.588585:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320731.588594:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5975 00000100:00080000:2.0:1767320731.588598:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320731.588606:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320731.588628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a580380 x1853163983143552/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320731.588637:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983143424:0@lo:400:kworker.0 00000100:00080000:2.0:1767320731.588681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320731.588685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320731.588687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320731.588691:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5975 00000100:00080000:2.0:1767320731.588693:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320731.588697:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320731.588713:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583800 x1853163983143808/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320731.588715:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320731.588727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320731.588731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320731.588733:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320731.588737:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5975 00000100:00080000:2.0:1767320731.588739:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320731.588744:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320731.588758:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581c00 x1853163983143936/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320731.588783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320731.588786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320731.588788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320731.588792:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5975 00000100:00080000:2.0:1767320731.588794:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320731.588797:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320731.588798:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320731.588809:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983144192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.588813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581f80 x1853163983144320/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320731.588850:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983143680:0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.588885:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320731.588907:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320731.588919:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320731.588927:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983144064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.588953:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320731.588959:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143424 00000100:00100000:3.0:1767320731.588968:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143424 00000100:00100000:2.0:1767320731.588986:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143680 00000100:00100000:2.0:1767320731.588992:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143680 00000100:00100000:3.0:1767320731.589037:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890 00000100:00100000:3.0:1767320731.589049:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983143424:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320731.589103:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983143424:12345-0@lo:400:kworker.0 Request processed in 56us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320731.589169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983143552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320731.589215:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320731.589429:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1891 00000100:00100000:2.0:1767320731.589438:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983143680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.589475:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983143680:12345-0@lo:400:kworker.0 Request processed in 38us (590us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320731.589482:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1891 00000100:00100000:2.0:1767320731.589496:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143552 00000100:00100000:2.0:1767320731.589500:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143552 00000100:00100000:2.0:1767320731.589553:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2501 00000100:00100000:2.0:1767320731.589559:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983143552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320731.589591:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320731.589658:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983143552:12345-0@lo:38:kworker.0 Request processed in 97us (442us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320731.589664:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2501 00010000:00080000:2.0:1767320731.589723:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a580000 x1853163983143680/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320747 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320731.589734:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983143680:0@lo:400:kworker.0 00010000:00080000:2.0:1767320731.589823:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581880 x1853163983143424/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320747 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320731.590481:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983143424:0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.590524:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144192 00000100:00100000:2.0:1767320731.590529:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144192 00000100:00100000:2.0:1767320731.590568:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144064 00000100:00100000:2.0:1767320731.590580:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144064 00000100:00100000:2.0:1767320731.590601:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1161 00000100:00100000:2.0:1767320731.590608:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983144192:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.590660:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983144192:12345-0@lo:400:kworker.0 Request processed in 53us (1755us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320731.590667:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1161 00000100:00100000:2.0:1767320731.590680:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1162 00000100:00100000:2.0:1767320731.590687:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983144064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.590775:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983144064:12345-0@lo:400:kworker.0 Request processed in 71us (1803us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320731.590790:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1162 00000100:00100000:2.0:1767320731.590848:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983144064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320731.590872:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983144192:0@lo:400:kworker.0 00000100:00080000:2.0:1767320731.590912:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320731.590915:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320731.590918:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320731.591002:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983143808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320731.591039:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320731.591084:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983143936:0@lo:38:kworker.0 00000100:00100000:1.0:1767320731.591142:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320731.591165:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983144320:0@lo:38:kworker.0 00000100:00100000:1.0:1767320731.591189:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320731.591220:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320731.591225:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320731.591228:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983143552:0@lo:38:kworker.0 00000100:00100000:1.0:1767320731.591248:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143936 00000100:00100000:1.0:1767320731.591252:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143936 00000100:00100000:1.0:1767320731.591291:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 650 00000100:00100000:1.0:1767320731.591296:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983143936:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320731.591311:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320731.591343:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983143936:12345-0@lo:38:kworker.0 Request processed in 45us (200us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320731.591360:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 650 00000100:00100000:1.0:1767320731.591381:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983143808 00000100:00100000:1.0:1767320731.591385:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983143808 00000100:00100000:1.0:1767320731.591402:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144320 00000100:00100000:1.0:1767320731.591405:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144320 00000100:00100000:1.0:1767320731.591415:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2502 00000100:00100000:1.0:1767320731.591419:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983143808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320731.591428:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320731.591453:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983143808:12345-0@lo:38:kworker.0 Request processed in 32us (413us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320731.591459:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2502 00000100:00100000:1.0:1767320731.591470:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2503 00000100:00100000:1.0:1767320731.591473:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983144320:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320731.591482:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320731.591505:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983144320:12345-0@lo:38:kworker.0 Request processed in 30us (315us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320731.591510:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2503 00000100:00080000:1.0:1767320731.591534:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320731.591538:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320731.591540:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983143808:0@lo:38:kworker.0 00000100:00080000:1.0:1767320731.591547:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320731.591550:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320731.591553:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983143936:0@lo:38:kworker.0 00000100:00080000:1.0:1767320731.591559:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320731.591562:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320731.591564:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983144320:0@lo:38:kworker.0 00000100:00100000:3.0:1767320731.595227:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890 00010000:00010000:3.0:1767320731.780610:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320732.804733:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320733.828449:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320734.148621:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320734.148646:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983144448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320734.148723:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320734.149412:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144448 00000100:00100000:0.0:1767320734.149422:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144448 00000100:00100000:0.0:1767320734.149501:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1264 00000100:00100000:0.0:1767320734.149523:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983144448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320734.149554:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320734.149609:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983144448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (885us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320734.149615:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1264 00000100:00080000:2.0:1767320734.149818:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320734.151024:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983144448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320734.151229:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320734.151242:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983144576:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320734.151309:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320734.151454:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144576 00000100:00100000:0.0:1767320734.151461:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144576 00000100:00100000:0.0:1767320734.151520:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265 00000100:00100000:0.0:1767320734.151529:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983144576:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320734.151554:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320734.151605:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983144576:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320734.151612:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265 00000100:00100000:3.0:1767320734.151685:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983144576:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320734.151731:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320734.603893:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320734.603944:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320734.604039:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698176 00000100:00100000:1.0:1767320734.604049:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698176 00000100:00100000:1.0:1767320734.604122:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698048 00000100:00100000:1.0:1767320734.604126:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698048 00000100:00100000:1.0:1767320734.604146:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1892 00000100:00100000:1.0:1767320734.604155:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915698176:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320734.604226:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915698176:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320734.604234:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1892 00000100:00100000:1.0:1767320734.604251:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2504 00000100:00100000:1.0:1767320734.604256:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915698048:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320734.604320:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320734.604346:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915698048:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 88us (401us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320734.604506:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2504 00000100:00080000:2.0:1767320734.605264:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320734.852500:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320735.878553:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320736.711815:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320736.711827:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320736.711896:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320736.711901:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320736.711905:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320736.711912:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5980 00000100:00080000:1.0:1767320736.711917:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320736.711924:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320736.711947:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163983144832/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320736.712027:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320736.712031:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320736.712033:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320736.712037:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5980 00000100:00080000:1.0:1767320736.712039:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320736.712043:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320736.712055:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163983145088/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320736.712063:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320736.712066:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320736.712068:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320736.712071:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5980 00000100:00080000:1.0:1767320736.712073:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320736.712076:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320736.712087:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967b80 x1853163983145216/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320736.712121:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320736.712125:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320736.712127:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320736.712131:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5980 00000100:00080000:1.0:1767320736.712133:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320736.712136:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320736.712160:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f965880 x1853163983145600/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320736.712241:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983145344:0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712519:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320736.712595:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320736.712602:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983144960:0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712629:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320736.712661:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144960 00000100:00100000:3.0:1767320736.712669:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144960 00000100:00100000:3.0:1767320736.712722:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1893 00000100:00100000:3.0:1767320736.712731:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983144960:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712791:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983144960:12345-0@lo:400:kworker.0 Request processed in 61us (160us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320736.712809:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1893 00000100:00080000:3.0:1767320736.712831:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320736.712855:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f964e00 x1853163983144960/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320752 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320736.712869:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983144960:0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712891:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983144704:0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712915:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320736.712938:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320736.712944:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983145472:0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.712964:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320736.712985:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144704 00000100:00100000:3.0:1767320736.712988:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144704 00000100:00100000:3.0:1767320736.713010:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894 00000100:00100000:3.0:1767320736.713015:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983144704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320736.713044:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983144704:12345-0@lo:400:kworker.0 Request processed in 27us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320736.713050:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894 00000100:00080000:3.0:1767320736.713062:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320736.713077:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965c00 x1853163983144704/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320752 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320736.713087:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983144704:0@lo:400:kworker.0 00000100:00100000:0.0:1767320736.713178:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983144832:0@lo:38:kworker.0 00000100:00100000:0.0:1767320736.713208:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320736.713240:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983145088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320736.713260:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320736.713307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983145216:0@lo:38:kworker.0 00000100:00100000:0.0:1767320736.713330:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320736.713350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983145600:0@lo:38:kworker.0 00000100:00100000:0.0:1767320736.713368:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320736.713389:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983144832 00000100:00100000:0.0:1767320736.713393:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983144832 00000100:00100000:0.0:1767320736.713406:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145088 00000100:00100000:0.0:1767320736.713408:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145088 00000100:00100000:0.0:1767320736.713415:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145600 00000100:00100000:0.0:1767320736.713417:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145600 00000100:00100000:0.0:1767320736.713425:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2505 00000100:00100000:0.0:1767320736.713431:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983144832:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320736.713449:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320736.713480:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983144832:12345-0@lo:38:kworker.0 Request processed in 48us (271us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320736.713486:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2505 00000100:00100000:0.0:1767320736.713497:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2506 00000100:00100000:0.0:1767320736.713500:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983145088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320736.713509:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320736.713531:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983145088:12345-0@lo:38:kworker.0 Request processed in 28us (269us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320736.713536:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2506 00000100:00100000:0.0:1767320736.713546:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2507 00000100:00100000:0.0:1767320736.713548:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983145600:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320736.713556:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320736.713577:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983145600:12345-0@lo:38:kworker.0 Request processed in 26us (207us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320736.713582:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2507 00000100:00080000:0.0:1767320736.713606:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320736.713611:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320736.713614:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983144832:0@lo:38:kworker.0 00000100:00080000:0.0:1767320736.713622:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320736.713625:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320736.713628:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983145088:0@lo:38:kworker.0 00000100:00080000:0.0:1767320736.713634:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320736.713638:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320736.713640:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983145600:0@lo:38:kworker.0 00000100:00100000:2.0:1767320736.713759:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145344 00000100:00100000:2.0:1767320736.713764:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145344 00000100:00100000:2.0:1767320736.713821:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145472 00000100:00100000:2.0:1767320736.713825:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145472 00000100:00100000:2.0:1767320736.713847:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1163 00000100:00100000:2.0:1767320736.713854:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983145344:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320736.713892:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983145344:12345-0@lo:400:kworker.0 Request processed in 39us (1374us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320736.713899:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1163 00000100:00100000:2.0:1767320736.713911:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1164 00000100:00100000:2.0:1767320736.713917:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983145472:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320736.713945:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983145472:12345-0@lo:400:kworker.0 Request processed in 28us (979us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320736.713950:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1164 00000100:00100000:2.0:1767320736.713979:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983145344:0@lo:400:kworker.0 00000100:00080000:3.0:1767320736.714042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320736.714045:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320736.714065:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983145472:0@lo:400:kworker.0 00000100:00100000:1.0:1767320736.716499:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145216 00000100:00100000:1.0:1767320736.716515:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145216 00000100:00100000:1.0:1767320736.716574:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 651 00000100:00100000:1.0:1767320736.716582:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983145216:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320736.716606:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320736.716702:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983145216:12345-0@lo:38:kworker.0 Request processed in 119us (3369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320736.716710:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 651 00000100:00080000:0.0:1767320736.716829:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320736.716834:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320736.716838:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983145216:0@lo:38:kworker.0 00010000:00010000:1.0:1767320736.900661:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320737.926917:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767320738.700826:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915698304 offset 0 length 520: 4 00000100:00100000:0.0:1767320738.702229:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320738.702425:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698432 00000100:00100000:1.0:1767320738.702436:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698432 00000100:00100000:2.0:1767320738.702644:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2508 00000100:00100000:2.0:1767320738.702653:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915698432:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320738.702674:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320738.702745:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915698432:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 91us (522us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320738.702753:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2508 00000100:00100000:3.0:1767320738.703148:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320738.703426:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698560 00000100:00100000:2.0:1767320738.703433:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698560 00000100:00100000:2.0:1767320738.703557:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1895 00000100:00100000:2.0:1767320738.703565:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915698560:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320738.703624:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915698560:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 59us (477us total) trans 0 rc 0/0 00000100:00080000:3.0:1767320738.704172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320738.704726:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1895 00010000:00010000:1.0:1767320738.948470:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320739.268592:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320739.268616:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983145728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320739.268686:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320739.268912:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145728 00000100:00100000:0.0:1767320739.268920:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145728 00000100:00100000:0.0:1767320739.268990:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1266 00000100:00100000:0.0:1767320739.269000:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983145728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320739.269076:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320739.269125:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983145728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 127us (438us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320739.269132:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1266 00000100:00080000:2.0:1767320739.269201:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320739.269237:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983145728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320739.269428:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320739.269438:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983145856:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320739.269467:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320739.269539:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983145856 00000100:00100000:0.0:1767320739.269543:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983145856 00000100:00100000:0.0:1767320739.269585:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1267 00000100:00100000:0.0:1767320739.269592:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983145856:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320739.269611:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320739.269646:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983145856:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320739.269652:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1267 00000100:00100000:0.0:1767320739.269679:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983145856:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320739.269734:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320739.973890:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320740.998930:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320741.828633:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320716/real 1767320716] req@ffff93460a911880 x1853163983139456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320741.828666:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a911880 x1853163983139456/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320741.828685:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320741.828695:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320741.828711:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983139456:0@lo:250:kworker.0 00000100:00080000:1.0:1767320741.828806:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320741.828811:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767320741.828814:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320741.828817:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5964 00000100:00080000:1.0:1767320741.828820:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320741.828825:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320741.828857:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967480 x1853163983145984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320741.828951:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320741.828954:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320741.828955:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320741.828958:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5985 00000100:00080000:1.0:1767320741.828960:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320741.828963:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320741.828977:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964700 x1853163983146240/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320741.828996:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320741.828998:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320741.829000:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320741.829002:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5985 00000100:00080000:1.0:1767320741.829003:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320741.829006:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320741.829014:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f967b80 x1853163983146496/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320741.829019:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320741.829020:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320741.829022:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320741.829024:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5985 00000100:00080000:1.0:1767320741.829025:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320741.829028:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320741.829033:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966d80 x1853163983146624/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320741.829045:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320741.829047:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320741.829048:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320741.829051:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5985 00000100:00080000:1.0:1767320741.829052:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320741.829054:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320741.829059:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163983147008/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320741.829112:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983145984:0@lo:250:kworker.0 00000400:00000100:0.0:1767320741.829214:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983145984 offset 0 length 520: 4 00000100:00100000:0.0:1767320741.829248:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983146240:0@lo:38:kworker.0 00000100:00100000:3.0:1767320741.830073:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983146880:0@lo:400:kworker.0 00000100:00100000:2.0:1767320741.830125:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983146112:0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830477:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320741.830544:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983146752:0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830569:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320741.830586:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320741.830590:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983146368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830612:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320741.830642:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146880 00000100:00100000:3.0:1767320741.830649:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146880 00000100:00100000:3.0:1767320741.830711:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146752 00000100:00100000:3.0:1767320741.830715:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146752 00000100:00100000:3.0:1767320741.830740:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1165 00000100:00100000:3.0:1767320741.830748:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983146880:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830791:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983146880:12345-0@lo:400:kworker.0 Request processed in 44us (315us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320741.830796:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1165 00000100:00100000:3.0:1767320741.830812:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1166 00000100:00100000:3.0:1767320741.830817:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983146752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830848:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983146752:12345-0@lo:400:kworker.0 Request processed in 31us (277us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320741.830853:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1166 00000100:00080000:3.0:1767320741.830872:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320741.830877:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320741.830911:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983146752:0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830933:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983146880:0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.830967:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146368 00000100:00100000:3.0:1767320741.830970:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146368 00000100:00100000:3.0:1767320741.831016:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896 00000100:00100000:3.0:1767320741.831021:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983146368:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320741.831052:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983146368:12345-0@lo:400:kworker.0 Request processed in 31us (438us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320741.831057:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896 00000100:00080000:3.0:1767320741.831070:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320741.831086:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f966300 x1853163983146368/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320757 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320741.831095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983146368:0@lo:400:kworker.0 00000100:00100000:2.0:1767320741.831709:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320741.831804:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146112 00000100:00100000:1.0:1767320741.831809:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146112 00000100:00100000:1.0:1767320741.831851:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1897 00000100:00100000:1.0:1767320741.831857:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983146112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320741.831898:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983146112:12345-0@lo:400:kworker.0 Request processed in 41us (199us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320741.831904:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1897 00010000:00080000:2.0:1767320741.831968:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965500 x1853163983146112/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320757 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320741.831975:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983146112:0@lo:400:kworker.0 00000100:00080000:3.0:1767320741.832020:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320741.832054:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320741.832098:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983146496:0@lo:38:kworker.0 00000100:00100000:0.0:1767320741.832115:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320741.832372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983146624:0@lo:38:kworker.0 00000100:00100000:0.0:1767320741.832392:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320741.832417:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983147008:0@lo:38:kworker.0 00000100:00100000:0.0:1767320741.832442:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320741.832553:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146624 00000100:00100000:3.0:1767320741.832557:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146624 00000100:00100000:3.0:1767320741.832574:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 652 00000100:00100000:3.0:1767320741.832578:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983146624:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320741.832594:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320741.832618:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983146624:12345-0@lo:38:kworker.0 Request processed in 39us (225us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320741.832622:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 652 00000100:00100000:2.0:1767320741.832667:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146240 00000100:00100000:2.0:1767320741.832672:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146240 00000100:00100000:1.0:1767320741.832709:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983146496 00000100:00100000:1.0:1767320741.832713:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983146496 00000100:00100000:1.0:1767320741.832740:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2509 00000100:00100000:1.0:1767320741.832745:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983146240:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320741.832752:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320741.832772:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983146240:12345-0@lo:38:kworker.0 Request processed in 28us (719us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320741.832776:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2509 00000100:00100000:1.0:1767320741.832784:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2510 00000100:00100000:1.0:1767320741.832786:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983146496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320741.832791:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320741.832805:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983146496:12345-0@lo:38:kworker.0 Request processed in 18us (689us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320741.832808:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2510 00000100:00080000:0.0:1767320741.832921:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320741.832927:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320741.832931:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983146240:0@lo:38:kworker.0 00000100:00080000:0.0:1767320741.832939:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320741.832943:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320741.832944:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983146496:0@lo:38:kworker.0 00000100:00080000:0.0:1767320741.832950:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320741.832952:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320741.832954:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983146624:0@lo:38:kworker.0 00000100:00100000:3.0:1767320741.832995:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147008 00000100:00100000:3.0:1767320741.832999:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147008 00000100:00100000:3.0:1767320741.833059:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2511 00000100:00100000:3.0:1767320741.833064:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983147008:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320741.833073:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320741.833113:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983147008:12345-0@lo:38:kworker.0 Request processed in 47us (670us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320741.833117:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2511 00000100:00080000:0.0:1767320741.833366:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320741.833370:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320741.833372:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983147008:0@lo:38:kworker.0 00010000:00010000:1.0:1767320742.021473:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320743.047990:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320743.820069:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320743.820245:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698688 00000100:00100000:1.0:1767320743.820257:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698688 00000100:00100000:1.0:1767320743.820361:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2512 00000100:00100000:1.0:1767320743.820379:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915698688:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320743.820404:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320743.820477:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915698688:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 96us (417us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320743.820485:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2512 00000100:00100000:0.0:1767320743.821616:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320743.821693:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698816 00000100:00100000:3.0:1767320743.821700:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698816 00000100:00100000:3.0:1767320743.821772:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1898 00000100:00100000:3.0:1767320743.821782:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915698816:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320743.821897:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915698816:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 116us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320743.821904:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1898 00000100:00080000:0.0:1767320743.822089:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320744.070981:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320744.388799:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983147136:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320744.388869:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320744.392396:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320744.392419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983147264:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320744.392495:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320744.392781:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147136 00000100:00100000:0.0:1767320744.392790:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147136 00000100:00100000:0.0:1767320744.392861:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147264 00000100:00100000:0.0:1767320744.392865:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147264 00000100:00100000:0.0:1767320744.392884:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268 00000100:00100000:0.0:1767320744.392894:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983147136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320744.392928:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320744.392979:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983147136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (4108us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320744.392987:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268 00000100:00100000:0.0:1767320744.393005:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269 00000100:00100000:0.0:1767320744.393012:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983147264:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320744.393028:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320744.393058:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983147264:12345-0@lo:13:osp-pre-0-1.0 Request processed in 46us (563us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320744.393064:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269 00000100:00100000:0.0:1767320744.393103:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983147136:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767320744.393183:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320744.393189:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320744.394817:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983147264:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320745.094885:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320746.116877:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320746.949100:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320746.949111:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320746.949182:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320746.949193:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320746.949196:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320746.949205:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5990 00000100:00080000:1.0:1767320746.949209:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320746.949215:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320746.949244:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964e00 x1853163983147520/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320746.949326:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320746.949330:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320746.949332:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320746.949336:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5990 00000100:00080000:1.0:1767320746.949340:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320746.949345:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320746.949359:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f966680 x1853163983147776/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320746.949366:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320746.949369:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320746.949371:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320746.949374:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5990 00000100:00080000:1.0:1767320746.949376:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320746.949380:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320746.949400:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964a80 x1853163983147904/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320746.949422:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320746.949425:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320746.949426:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320746.949429:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5990 00000100:00080000:1.0:1767320746.949431:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320746.949434:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320746.949442:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f964700 x1853163983148288/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320746.949481:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983148032:0@lo:400:kworker.0 00000100:00100000:1.0:1767320746.949573:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320746.949639:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320746.949645:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983148160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320746.949649:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983147648:0@lo:400:kworker.0 00000100:00100000:3.0:1767320746.949690:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320746.949696:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320746.949724:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320746.949732:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983147392:0@lo:400:kworker.0 00000100:00100000:1.0:1767320746.949742:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148032 00000100:00100000:1.0:1767320746.949751:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148032 00000100:00100000:3.0:1767320746.949758:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320746.949805:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148160 00000100:00100000:3.0:1767320746.949810:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148160 00000100:00100000:1.0:1767320746.949831:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1167 00000100:00100000:2.0:1767320746.949901:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147648 00000100:00100000:2.0:1767320746.949906:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147648 00000100:00100000:2.0:1767320746.949958:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1899 00000100:00100000:2.0:1767320746.949967:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983147648:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320746.950018:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983147648:12345-0@lo:400:kworker.0 Request processed in 54us (324us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320746.950025:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1899 00000100:00100000:2.0:1767320746.950038:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147392 00000100:00100000:2.0:1767320746.950041:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147392 00000100:00100000:2.0:1767320746.950061:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900 00000100:00100000:2.0:1767320746.950065:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983147392:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320746.950092:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983147392:12345-0@lo:400:kworker.0 Request processed in 27us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320746.950098:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900 00010000:00080000:2.0:1767320746.950131:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f965c00 x1853163983147392/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320762 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320746.950144:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983147392:0@lo:400:kworker.0 00010000:00080000:2.0:1767320746.950169:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f967800 x1853163983147648/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320762 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320746.950176:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983147648:0@lo:400:kworker.0 00000100:00100000:0.0:1767320746.950443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983147520:0@lo:38:kworker.0 00000100:00100000:0.0:1767320746.950488:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320746.950514:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983147776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320746.950533:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320746.950548:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983147904:0@lo:38:kworker.0 00000100:00100000:0.0:1767320746.950591:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320746.950623:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983148288:0@lo:38:kworker.0 00000100:00100000:0.0:1767320746.950643:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320746.950669:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147904 00000100:00100000:0.0:1767320746.950673:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147904 00000100:00100000:0.0:1767320746.950695:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 653 00000100:00100000:0.0:1767320746.950701:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983147904:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320746.950733:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320746.950769:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983147904:12345-0@lo:38:kworker.0 Request processed in 67us (176us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320746.950775:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 653 00000100:00100000:0.0:1767320746.950793:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147520 00000100:00100000:0.0:1767320746.950797:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147520 00000100:00100000:0.0:1767320746.950814:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983147776 00000100:00100000:0.0:1767320746.950817:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983147776 00000100:00100000:0.0:1767320746.950827:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148288 00000100:00100000:0.0:1767320746.950830:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148288 00000100:00100000:0.0:1767320746.950839:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2513 00000100:00100000:0.0:1767320746.950842:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983147520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320746.950852:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320746.950878:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983147520:12345-0@lo:38:kworker.0 Request processed in 34us (388us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320746.950883:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2513 00000100:00100000:0.0:1767320746.950893:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2514 00000100:00100000:0.0:1767320746.950896:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983147776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320746.950904:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320746.950938:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983147776:12345-0@lo:38:kworker.0 Request processed in 39us (403us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320746.950943:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2514 00000100:00100000:0.0:1767320746.950954:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2515 00000100:00100000:0.0:1767320746.950958:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983148288:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320746.950965:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320746.950988:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983148288:12345-0@lo:38:kworker.0 Request processed in 29us (344us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320746.950992:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2515 00000100:00080000:0.0:1767320746.951023:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320746.951028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320746.951031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983147520:0@lo:38:kworker.0 00000100:00080000:0.0:1767320746.951039:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320746.951042:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320746.951045:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983147776:0@lo:38:kworker.0 00000100:00080000:0.0:1767320746.951051:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320746.951054:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320746.951056:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983147904:0@lo:38:kworker.0 00000100:00080000:0.0:1767320746.951061:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320746.951064:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320746.951066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f964700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983148288:0@lo:38:kworker.0 00000100:00080000:0.0:1767320746.951102:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320746.951107:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320746.951108:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320746.951419:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983148032:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320746.951474:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983148032:12345-0@lo:400:kworker.0 Request processed in 61us (1900us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320746.951589:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983148032:0@lo:400:kworker.0 00000100:00100000:3.0:1767320746.952946:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1168 00000100:00100000:3.0:1767320746.953216:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983148160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320746.954481:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983148160:12345-0@lo:400:kworker.0 Request processed in 1267us (4784us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320746.954493:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1168 00000100:00100000:0.0:1767320746.954663:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983148160:0@lo:400:kworker.0 00000100:00100000:1.0:1767320746.961299:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1167 00010000:00010000:3.0:1767320747.140593:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320748.170683:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320748.939799:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320748.939900:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320748.941667:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915698944 00000100:00100000:2.0:1767320748.941688:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915698944 00000100:00100000:0.0:1767320748.941828:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699072 00000100:00100000:0.0:1767320748.941833:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699072 00000100:00100000:0.0:1767320748.941896:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1901 00000100:00100000:0.0:1767320748.941907:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915699072:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320748.941984:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915699072:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 79us (2084us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320748.941992:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1901 00000100:00100000:0.0:1767320748.942010:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2516 00000100:00100000:0.0:1767320748.942014:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915698944:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320748.942049:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320748.942087:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915698944:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 71us (2309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320748.942094:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2516 00000100:00080000:3.0:1767320748.942338:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320749.193503:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320749.508965:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320749.508990:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983148416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320749.509061:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320749.509169:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148416 00000100:00100000:2.0:1767320749.509178:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148416 00000100:00100000:2.0:1767320749.509243:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1270 00000100:00100000:2.0:1767320749.509253:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983148416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320749.509305:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320749.509351:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983148416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (289us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320749.509357:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1270 00000100:00080000:0.0:1767320749.510135:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320749.510154:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320749.510167:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983148544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320749.510207:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320749.510265:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983148416:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320749.510402:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148544 00000100:00100000:3.0:1767320749.510408:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148544 00000100:00100000:1.0:1767320749.510851:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271 00000100:00100000:1.0:1767320749.510862:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983148544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320749.510884:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320749.510925:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983148544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (717us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320749.510932:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271 00000100:00080000:0.0:1767320749.511381:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320749.511414:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983148544:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320750.212560:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320751.236962:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320752.068587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.068598:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320752.068693:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983148672:0@lo:400:kworker.0 00000100:00100000:0.0:1767320752.068782:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320752.068875:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148672 00000100:00100000:1.0:1767320752.068884:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148672 00000100:00100000:1.0:1767320752.068967:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902 00000100:00100000:1.0:1767320752.068977:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983148672:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320752.069052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.069056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.069060:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.069065:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.069069:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.069075:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320752.069097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582a00 x1853163983148800/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320752.069138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.069142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.069144:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.069147:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.069150:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.069154:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:3.0:1767320752.069156:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320752.069166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581c00 x1853163983149056/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.069169:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1767320752.069180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767320752.069184:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983148928:0@lo:400:kworker.0 00000100:00080000:2.0:1767320752.069185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.069188:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.069194:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.069197:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.069200:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320752.069213:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583800 x1853163983149184/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.069223:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.069251:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983149312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069310:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.069334:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148928 00000100:00100000:3.0:1767320752.069338:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148928 00000100:00100000:3.0:1767320752.069366:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1903 00000100:00100000:3.0:1767320752.069372:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983148928:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069401:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983148928:12345-0@lo:400:kworker.0 Request processed in 29us (179us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320752.069408:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1903 00000100:00080000:3.0:1767320752.069429:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320752.069462:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a581f80 x1853163983148928/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320752.069476:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983148928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069507:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983149440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069540:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.069575:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149312 00000100:00100000:3.0:1767320752.069578:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149312 00000100:00100000:3.0:1767320752.069601:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149440 00000100:00100000:3.0:1767320752.069603:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149440 00000100:00100000:3.0:1767320752.069618:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1169 00000100:00100000:3.0:1767320752.069623:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983149312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069651:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983149312:12345-0@lo:400:kworker.0 Request processed in 27us (341us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320752.069657:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1169 00000100:00100000:3.0:1767320752.069666:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1170 00000100:00100000:3.0:1767320752.069671:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983149440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069706:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983149440:12345-0@lo:400:kworker.0 Request processed in 34us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320752.069711:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1170 00000100:00080000:3.0:1767320752.069731:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320752.069733:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320752.069748:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983149440:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069766:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983149312:0@lo:400:kworker.0 00000100:00100000:3.0:1767320752.069794:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983148800:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.069820:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.069849:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983149056:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.069869:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.069883:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983149184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.070104:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320752.070144:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983148800 00000100:00100000:3.0:1767320752.070149:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983148800 00000100:00100000:3.0:1767320752.070171:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149056 00000100:00100000:3.0:1767320752.070175:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149056 00000100:00100000:3.0:1767320752.070354:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2517 00000100:00100000:3.0:1767320752.070360:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983148800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320752.070378:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320752.070416:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983148800:12345-0@lo:38:kworker.0 Request processed in 55us (596us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320752.070423:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2517 00000100:00100000:3.0:1767320752.070435:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2518 00000100:00100000:3.0:1767320752.070439:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983149056:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320752.070448:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320752.070471:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983149056:12345-0@lo:38:kworker.0 Request processed in 30us (602us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320752.070477:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2518 00000100:00080000:3.0:1767320752.070506:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.070524:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.070528:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983148800:0@lo:38:kworker.0 00000100:00080000:3.0:1767320752.070538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.070543:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.070546:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983149056:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.070577:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149184 00000100:00100000:3.0:1767320752.070581:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149184 00000100:00100000:3.0:1767320752.070601:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 654 00000100:00100000:3.0:1767320752.070605:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983149184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320752.070615:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320752.070645:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983149184:12345-0@lo:38:kworker.0 Request processed in 38us (541us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320752.070651:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 654 00000100:00080000:3.0:1767320752.070673:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320752.070677:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320752.070680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983149184:0@lo:38:kworker.0 00000100:00080000:2.0:1767320752.073167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320752.073178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320752.073182:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320752.073189:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 5995 00000100:00080000:2.0:1767320752.073195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320752.073202:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320752.073244:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd500 x1853163983149568/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320752.073374:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983149568:0@lo:38:kworker.0 00000100:00100000:3.0:1767320752.073458:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320752.073561:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149568 00000100:00100000:2.0:1767320752.073569:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149568 00000100:00100000:2.0:1767320752.073608:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2519 00000100:00100000:2.0:1767320752.073615:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983149568:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320752.073634:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320752.073681:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983149568:12345-0@lo:38:kworker.0 Request processed in 65us (230us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320752.073689:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2519 00000100:00080000:2.0:1767320752.073725:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320752.073730:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320752.073733:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983149568:0@lo:38:kworker.0 00000100:00100000:1.0:1767320752.076916:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983148672:12345-0@lo:400:kworker.0 Request processed in 7920us (8113us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320752.076932:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902 00010000:00080000:0.0:1767320752.077082:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583480 x1853163983148672/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320752.077100:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983148672:0@lo:400:kworker.0 00010000:00010000:1.0:1767320752.261485:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320753.287805:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320754.061231:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320754.061350:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320754.061435:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699328 00000100:00100000:3.0:1767320754.061444:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699328 00000100:00100000:3.0:1767320754.061556:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699200 00000100:00100000:3.0:1767320754.061560:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699200 00000100:00100000:3.0:1767320754.061604:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1904 00000100:00100000:3.0:1767320754.061614:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915699328:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320754.061708:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915699328:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (484us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320754.061715:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1904 00000100:00100000:3.0:1767320754.061735:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2520 00000100:00100000:3.0:1767320754.061740:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699200:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320754.061759:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320754.061786:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699200:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (436us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320754.061792:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2520 00000100:00080000:0.0:1767320754.062717:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320754.316330:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320754.629019:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320754.629020:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320754.629046:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983149824:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320754.629046:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983149696:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320754.629122:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320754.629156:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320754.629200:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149824 00000100:00100000:1.0:1767320754.629206:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149824 00000100:00100000:2.0:1767320754.629211:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149696 00000100:00100000:2.0:1767320754.629217:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149696 00000100:00100000:1.0:1767320754.629249:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272 00000100:00100000:1.0:1767320754.629256:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983149824:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320754.629309:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320754.629315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767320754.629315:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320754.629354:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983149824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320754.629420:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1273 00000100:00100000:2.0:1767320754.629432:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983149696:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320754.629450:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320754.629589:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983149696:12345-0@lo:13:osp-pre-0-1.0 Request processed in 160us (432us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320754.629597:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1273 00000100:00100000:3.0:1767320754.629748:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983149696:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320754.629791:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f965180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983149824:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320754.632447:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272 00010000:00010000:1.0:1767320755.333366:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320756.359310:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320757.188430:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320757.188441:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320757.188510:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320757.188515:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320757.188519:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320757.188527:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6000 00000100:00080000:2.0:1767320757.188532:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320757.188539:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320757.188621:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163983150080/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320757.188669:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320757.188672:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320757.188674:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320757.188678:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6000 00000100:00080000:2.0:1767320757.188681:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320757.188685:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320757.188710:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983150336/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320757.188715:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983149952:0@lo:400:kworker.0 00000100:00080000:2.0:1767320757.188719:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320757.188722:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320757.188724:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320757.188727:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6000 00000100:00080000:2.0:1767320757.188729:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320757.188731:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320757.188745:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163983150464/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320757.188764:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320757.188767:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320757.188768:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320757.188771:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6000 00000100:00080000:2.0:1767320757.188774:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320757.188776:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320757.188786:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de680 x1853163983150848/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320757.188791:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320757.188808:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983150208:0@lo:400:kworker.0 00000100:00100000:2.0:1767320757.188834:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320757.188846:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320757.188851:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983150720:0@lo:400:kworker.0 00000100:00100000:2.0:1767320757.188867:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320757.188873:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983150592:0@lo:400:kworker.0 00000100:00100000:3.0:1767320757.188875:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320757.188894:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320757.189038:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983149952 00000100:00100000:0.0:1767320757.189046:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983149952 00000100:00100000:0.0:1767320757.189096:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150208 00000100:00100000:0.0:1767320757.189099:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150208 00000100:00100000:0.0:1767320757.189121:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1905 00000100:00100000:0.0:1767320757.189130:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983149952:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.189183:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983149952:12345-0@lo:400:kworker.0 Request processed in 54us (392us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320757.189190:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1905 00000100:00100000:0.0:1767320757.189222:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906 00000100:00100000:0.0:1767320757.189229:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983150208:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.189256:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983150208:12345-0@lo:400:kworker.0 Request processed in 28us (422us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320757.189261:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906 00000100:00080000:0.0:1767320757.189295:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320757.189910:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddf80 x1853163983150208/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320773 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320757.189925:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983150208:0@lo:400:kworker.0 00010000:00080000:2.0:1767320757.189982:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dea00 x1853163983149952/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320773 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320757.189991:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983149952:0@lo:400:kworker.0 00000100:00100000:1.0:1767320757.190510:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983150080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320757.190597:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.190639:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983150336:0@lo:38:kworker.0 00000100:00100000:1.0:1767320757.190664:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.190682:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983150464:0@lo:38:kworker.0 00000100:00100000:1.0:1767320757.190719:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.190745:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983150848:0@lo:38:kworker.0 00000100:00100000:1.0:1767320757.190767:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320757.190819:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150464 00000100:00100000:1.0:1767320757.190825:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150464 00000100:00100000:1.0:1767320757.190858:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767320757.190864:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983150464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320757.190883:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320757.190918:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983150464:12345-0@lo:38:kworker.0 Request processed in 53us (198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320757.190925:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 655 00000100:00100000:1.0:1767320757.190949:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150080 00000100:00100000:1.0:1767320757.190953:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150080 00000100:00100000:1.0:1767320757.190974:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150336 00000100:00100000:1.0:1767320757.190977:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150336 00000100:00100000:1.0:1767320757.190989:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150848 00000100:00100000:1.0:1767320757.190991:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150848 00000100:00100000:1.0:1767320757.191001:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2521 00000100:00100000:1.0:1767320757.191005:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150080:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320757.191016:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320757.191042:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150080:12345-0@lo:38:kworker.0 Request processed in 35us (445us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320757.191050:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2521 00000100:00100000:1.0:1767320757.191061:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2522 00000100:00100000:1.0:1767320757.191065:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150336:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320757.191073:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320757.191097:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150336:12345-0@lo:38:kworker.0 Request processed in 30us (432us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320757.191103:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2522 00000100:00100000:1.0:1767320757.191116:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2523 00000100:00100000:1.0:1767320757.191119:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320757.191127:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320757.191149:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983150848:12345-0@lo:38:kworker.0 Request processed in 29us (381us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320757.191155:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2523 00000100:00080000:1.0:1767320757.191202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320757.191208:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320757.191211:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983150080:0@lo:38:kworker.0 00000100:00080000:1.0:1767320757.191220:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320757.191224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320757.191226:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983150336:0@lo:38:kworker.0 00000100:00080000:1.0:1767320757.191233:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320757.191236:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320757.191238:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983150464:0@lo:38:kworker.0 00000100:00080000:1.0:1767320757.191244:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320757.191247:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320757.191249:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983150848:0@lo:38:kworker.0 00000100:00100000:0.0:1767320757.196305:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150720 00000100:00100000:3.0:1767320757.196311:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150592 00000100:00100000:0.0:1767320757.196318:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150720 00000100:00100000:3.0:1767320757.196319:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150592 00000100:00100000:0.0:1767320757.196450:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1171 00000100:00100000:0.0:1767320757.196459:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983150720:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320757.196557:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983150720:12345-0@lo:400:kworker.0 Request processed in 100us (7680us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320757.196562:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1171 00000100:00100000:2.0:1767320757.196675:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983150720:0@lo:400:kworker.0 00000100:00100000:3.0:1767320757.198318:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1172 00000100:00100000:3.0:1767320757.198333:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983150592:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320757.198662:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983150592:12345-0@lo:400:kworker.0 Request processed in 333us (9765us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320757.198670:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1172 00000100:00100000:2.0:1767320757.198707:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983150592:0@lo:400:kworker.0 00000100:00080000:1.0:1767320757.203368:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320757.203379:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:1.0:1767320757.383456:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320758.406801:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320759.182939:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320759.183417:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320759.183589:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699584 00000100:00100000:1.0:1767320759.183603:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699584 00000100:00100000:1.0:1767320759.183727:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699456 00000100:00100000:1.0:1767320759.183732:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699456 00000100:00100000:1.0:1767320759.183761:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1907 00000100:00100000:3.0:1767320759.183772:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2524 00000100:00100000:1.0:1767320759.183775:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915699584:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320759.183785:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699456:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320759.183813:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320759.183910:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699456:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 126us (493us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320759.183924:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2524 00000100:00100000:1.0:1767320759.183926:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915699584:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 152us (998us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320759.183936:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1907 00000100:00080000:0.0:1767320759.184078:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320759.429244:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320759.751695:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320759.751737:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983150976:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320759.751850:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320759.752691:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983151104:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320759.752746:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320759.752935:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983150976 00000100:00100000:1.0:1767320759.752949:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983150976 00000100:00100000:1.0:1767320759.753050:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151104 00000100:00100000:1.0:1767320759.753055:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151104 00000100:00100000:1.0:1767320759.753081:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1274 00000100:00100000:1.0:1767320759.753093:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983150976:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320759.753138:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320759.753200:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983150976:12345-0@lo:13:osp-pre-0-1.0 Request processed in 109us (1350us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320759.753400:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320759.753407:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320759.753443:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275 00000100:00100000:2.0:1767320759.753454:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983151104:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320759.753475:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320759.753516:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983151104:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (769us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320759.753524:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275 00000100:00100000:2.0:1767320759.753590:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f966680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983151104:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320759.753668:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1274 00000100:00100000:0.0:1767320759.753789:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983150976:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320760.453725:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320761.476627:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320762.309885:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320762.309901:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320762.309995:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320762.310000:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320762.310004:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320762.310014:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6005 00000100:00080000:2.0:1767320762.310019:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320762.310028:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320762.310057:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddf80 x1853163983151360/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320762.310090:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983151232:0@lo:400:kworker.0 00000100:00080000:2.0:1767320762.310111:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320762.310116:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320762.310118:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320762.310122:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6005 00000100:00080000:2.0:1767320762.310126:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320762.310131:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320762.310149:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd180 x1853163983151616/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320762.310159:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320762.310163:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320762.310165:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320762.310169:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6005 00000100:00080000:2.0:1767320762.310174:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320762.310181:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320762.310184:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320762.310202:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163983151744/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320762.310228:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320762.310231:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320762.310233:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320762.310237:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6005 00000100:00080000:2.0:1767320762.310240:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320762.310244:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320762.310259:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df800 x1853163983152128/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320762.310319:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151232 00000100:00100000:2.0:1767320762.310357:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151232 00000100:00100000:1.0:1767320762.310428:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983151360:0@lo:38:kworker.0 00000100:00100000:2.0:1767320762.310440:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908 00000100:00100000:2.0:1767320762.310455:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983151232:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320762.310515:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983152000:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.310613:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983151488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.316673:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.316731:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983151616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320762.316758:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.316774:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983151744:0@lo:38:kworker.0 00000100:00100000:1.0:1767320762.316794:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.316811:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983152128:0@lo:38:kworker.0 00000100:00100000:1.0:1767320762.316828:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320762.316871:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151744 00000100:00100000:1.0:1767320762.316878:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151744 00000100:00100000:1.0:1767320762.316909:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767320762.316915:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983151744:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320762.316936:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320762.316971:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983151744:12345-0@lo:38:kworker.0 Request processed in 55us (175us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320762.316977:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 656 00000100:00100000:1.0:1767320762.316996:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151360 00000100:00100000:1.0:1767320762.316999:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151360 00000100:00100000:1.0:1767320762.317008:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151616 00000100:00100000:1.0:1767320762.317010:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151616 00000100:00100000:1.0:1767320762.317015:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152128 00000100:00100000:1.0:1767320762.317017:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152128 00000100:00100000:1.0:1767320762.317024:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2525 00000100:00100000:1.0:1767320762.317028:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983151360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320762.317037:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320762.317056:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983151360:12345-0@lo:38:kworker.0 Request processed in 27us (391us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320762.317061:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2525 00000100:00100000:1.0:1767320762.317070:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2526 00000100:00100000:1.0:1767320762.317073:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983151616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320762.317081:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320762.317095:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983151616:12345-0@lo:38:kworker.0 Request processed in 21us (337us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320762.317100:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2526 00000100:00100000:1.0:1767320762.317108:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2527 00000100:00100000:1.0:1767320762.317111:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983152128:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320762.317118:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320762.317134:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983152128:12345-0@lo:38:kworker.0 Request processed in 22us (306us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320762.317139:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2527 00000100:00080000:1.0:1767320762.317174:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320762.317178:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320762.317182:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983151360:0@lo:38:kworker.0 00000100:00080000:1.0:1767320762.317189:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320762.317192:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320762.317195:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983151616:0@lo:38:kworker.0 00000100:00080000:1.0:1767320762.317202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320762.317205:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320762.317208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983151744:0@lo:38:kworker.0 00000100:00080000:1.0:1767320762.317214:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320762.317217:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320762.317220:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983152128:0@lo:38:kworker.0 00000100:00100000:1.0:1767320762.317266:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983151872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.317306:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320762.317366:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320762.319916:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151872 00000100:00100000:1.0:1767320762.319933:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151872 00000100:00100000:1.0:1767320762.320027:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1173 00000100:00100000:1.0:1767320762.320038:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983151872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320762.320136:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983151872:12345-0@lo:400:kworker.0 Request processed in 100us (2828us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320762.320143:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1173 00000100:00080000:1.0:1767320762.320172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320762.320206:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983151872:0@lo:400:kworker.0 00000100:00100000:2.0:1767320762.320984:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983151232:12345-0@lo:400:kworker.0 Request processed in 10523us (10798us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320762.321003:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908 00000100:00100000:3.0:1767320762.321658:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320762.321747:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152000 00000100:00100000:3.0:1767320762.321758:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152000 00000100:00100000:3.0:1767320762.321836:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1174 00000100:00100000:3.0:1767320762.321847:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983152000:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320762.321909:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983152000:12345-0@lo:400:kworker.0 Request processed in 64us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320762.321916:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1174 00000100:00100000:3.0:1767320762.321963:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983152000:0@lo:400:kworker.0 00000100:00080000:1.0:1767320762.322057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00080000:2.0:1767320762.322555:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dea00 x1853163983151232/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320778 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320762.322571:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983151232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320762.324301:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320762.325949:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983151488 00000100:00100000:2.0:1767320762.325961:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983151488 00000100:00100000:2.0:1767320762.326046:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1909 00000100:00100000:2.0:1767320762.326059:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983151488:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320762.326091:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320762.326159:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983151488:12345-0@lo:400:kworker.0 Request processed in 102us (1863us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320762.326168:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1909 00010000:00080000:0.0:1767320762.326376:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dfb80 x1853163983151488/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320778 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320762.326403:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983151488:0@lo:400:kworker.0 00010000:00010000:3.0:1767320762.500548:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320763.525977:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320764.305725:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320764.305777:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320764.305807:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915699968 offset 0 length 520: 4 00000100:00100000:1.0:1767320764.305970:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699712 00000100:00100000:1.0:1767320764.305983:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699712 00000100:00100000:1.0:1767320764.306036:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915699840 00000100:00100000:1.0:1767320764.306039:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915699840 00000100:00100000:1.0:1767320764.306099:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1910 00000100:00100000:1.0:1767320764.306111:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915699840:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320764.306202:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915699840:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 93us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320764.306210:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1910 00000100:00100000:1.0:1767320764.306229:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2528 00000100:00100000:1.0:1767320764.306234:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699712:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320764.306256:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320764.306353:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915699712:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 117us (643us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320764.306360:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2528 00000100:00080000:0.0:1767320764.306634:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320764.549645:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320764.872267:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983152256:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320764.872365:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320764.872434:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152256 00000100:00100000:3.0:1767320764.872443:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152256 00000100:00080000:0.0:1767320764.872472:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320764.872499:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1276 00000100:00100000:3.0:1767320764.872508:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983152256:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320764.872550:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320764.872601:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983152256:12345-0@lo:13:osp-pre-0-1.0 Request processed in 95us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320764.872608:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1276 00000100:00100000:1.0:1767320764.872631:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320764.872641:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983152384:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320764.872673:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320764.872699:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983152256:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320764.872805:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152384 00000100:00100000:3.0:1767320764.872809:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152384 00000100:00100000:3.0:1767320764.872851:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277 00000100:00100000:3.0:1767320764.872858:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983152384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320764.872874:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320764.872903:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983152384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (230us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320764.872909:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277 00000100:00080000:0.0:1767320764.872960:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320764.873019:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983152384:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320765.573739:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320766.404420:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320741/real 1767320741] req@ffff93463f967480 x1853163983145984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320766 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320766.404454:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463f967480 x1853163983145984/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320766 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320766.404474:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320766.404501:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320766.404506:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f967480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983145984:0@lo:250:kworker.0 00000100:00080000:2.0:1767320766.404596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320766.404602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320766.404605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320766.404610:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 5990 00000100:00080000:2.0:1767320766.404615:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320766.404622:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320766.404651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de680 x1853163983152512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320766.404713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320766.404717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320766.404718:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320766.404723:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320766.404725:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320766.404729:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320766.404742:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983152768/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320766.404768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320766.404771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320766.404773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320766.404776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320766.404778:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320766.404782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320766.404796:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd500 x1853163983153024/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320766.404804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320766.404807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320766.404809:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320766.404812:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320766.404814:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320766.404818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320766.404829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163983153152/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320766.404849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320766.404852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320766.404854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320766.404857:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6010 00000100:00080000:2.0:1767320766.404859:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320766.404863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320766.404873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de300 x1853163983153536/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320766.404902:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983153280:0@lo:400:kworker.0 00000100:00100000:2.0:1767320766.404960:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320766.405015:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320766.405020:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983152896:0@lo:400:kworker.0 00000100:00100000:2.0:1767320766.405044:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320766.405126:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983153408:0@lo:400:kworker.0 00000100:00100000:0.0:1767320766.405155:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320766.405194:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320766.405201:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983152640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320766.405223:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320766.405295:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152896 00000100:00100000:0.0:1767320766.405303:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152896 00000100:00100000:0.0:1767320766.405353:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152640 00000100:00100000:0.0:1767320766.405357:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152640 00000100:00100000:0.0:1767320766.405382:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1911 00000100:00100000:0.0:1767320766.405391:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983152896:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320766.405449:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983152896:12345-0@lo:400:kworker.0 Request processed in 59us (403us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320766.405456:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1911 00000100:00100000:0.0:1767320766.405471:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912 00000100:00100000:0.0:1767320766.405477:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983152640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320766.405508:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983152640:12345-0@lo:400:kworker.0 Request processed in 31us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320766.405514:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912 00000100:00080000:0.0:1767320766.405533:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320766.405569:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc000 x1853163983152640/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320782 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320766.405583:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983152640:0@lo:400:kworker.0 00010000:00080000:0.0:1767320766.405606:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382df100 x1853163983152896/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320782 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320766.405615:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983152896:0@lo:400:kworker.0 00000100:00100000:3.0:1767320766.405870:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153280 00000100:00100000:3.0:1767320766.405876:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153280 00000100:00100000:3.0:1767320766.405920:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153408 00000100:00100000:3.0:1767320766.405923:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153408 00000100:00100000:3.0:1767320766.405942:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1175 00000100:00100000:3.0:1767320766.405948:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983153280:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320766.405990:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983153280:12345-0@lo:400:kworker.0 Request processed in 41us (1029us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320766.405996:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1175 00000100:00100000:3.0:1767320766.406009:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1176 00000100:00100000:3.0:1767320766.406014:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983153408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320766.406042:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983153408:12345-0@lo:400:kworker.0 Request processed in 28us (886us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320766.406141:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1176 00000100:00100000:3.0:1767320766.406193:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983153408:0@lo:400:kworker.0 00000100:00100000:3.0:1767320766.406221:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983153280:0@lo:400:kworker.0 00000100:00100000:1.0:1767320766.406337:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983152512:0@lo:250:kworker.0 00000400:00000100:1.0:1767320766.406367:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983152512 offset 0 length 520: 4 00000100:00100000:1.0:1767320766.406390:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983152768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320766.406412:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320766.406438:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983153024:0@lo:38:kworker.0 00000100:00100000:1.0:1767320766.406457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320766.406472:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983153152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320766.406491:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320766.406523:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983153536:0@lo:38:kworker.0 00000100:00100000:1.0:1767320766.406543:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320766.406586:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153152 00000100:00100000:1.0:1767320766.406591:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153152 00000100:00100000:1.0:1767320766.406613:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 657 00000100:00100000:1.0:1767320766.406618:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983153152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320766.406635:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320766.406669:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983153152:12345-0@lo:38:kworker.0 Request processed in 50us (176us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320766.406674:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 657 00000100:00100000:1.0:1767320766.406697:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983152768 00000100:00100000:1.0:1767320766.406701:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983152768 00000100:00100000:1.0:1767320766.406718:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2529 00000100:00100000:1.0:1767320766.406722:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983152768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320766.406731:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320766.406757:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983152768:12345-0@lo:38:kworker.0 Request processed in 33us (344us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320766.406762:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2529 00000100:00100000:1.0:1767320766.406770:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153024 00000100:00100000:1.0:1767320766.406773:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153024 00000100:00100000:1.0:1767320766.406787:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153536 00000100:00100000:1.0:1767320766.406789:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153536 00000100:00100000:1.0:1767320766.406798:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2530 00000100:00100000:1.0:1767320766.406801:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983153024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320766.406809:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320766.406832:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983153024:12345-0@lo:38:kworker.0 Request processed in 30us (374us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320766.406837:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2530 00000100:00100000:1.0:1767320766.406848:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2531 00000100:00100000:1.0:1767320766.406851:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983153536:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320766.406859:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320766.406878:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983153536:12345-0@lo:38:kworker.0 Request processed in 26us (335us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320766.406884:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2531 00000100:00080000:1.0:1767320766.406917:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320766.406922:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320766.406925:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983152768:0@lo:38:kworker.0 00000100:00080000:1.0:1767320766.406932:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320766.406935:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320766.406937:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983153024:0@lo:38:kworker.0 00000100:00080000:1.0:1767320766.406946:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320766.406949:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320766.406952:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983153152:0@lo:38:kworker.0 00000100:00080000:1.0:1767320766.406958:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320766.406961:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320766.406963:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983153536:0@lo:38:kworker.0 00000100:00080000:0.0:1767320766.407335:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320766.407339:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320766.596519:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320767.620645:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320768.644808:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320769.422320:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320769.422459:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320769.422633:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700224 00000100:00100000:1.0:1767320769.422646:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700224 00000100:00100000:1.0:1767320769.422721:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700096 00000100:00100000:1.0:1767320769.422734:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700096 00000100:00100000:1.0:1767320769.422759:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1913 00000100:00100000:1.0:1767320769.422769:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915700224:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320769.422845:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915700224:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 77us (567us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320769.422854:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1913 00000100:00100000:1.0:1767320769.422872:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2532 00000100:00100000:1.0:1767320769.422877:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915700096:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320769.422897:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320769.422924:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915700096:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (465us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320769.422931:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2532 00000100:00080000:0.0:1767320769.423637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320769.669312:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320769.988561:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320769.988583:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983153664:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320769.988663:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320769.989313:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153664 00000100:00100000:1.0:1767320769.989324:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153664 00000100:00100000:1.0:1767320769.989399:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1278 00000100:00100000:1.0:1767320769.989409:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983153664:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320769.989441:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320769.989452:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983153792:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320769.989493:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320769.989505:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983153664:12345-0@lo:13:osp-pre-0-1.0 Request processed in 96us (840us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320769.989513:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1278 00000100:00100000:3.0:1767320769.989528:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153792 00000100:00100000:3.0:1767320769.989533:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153792 00000100:00100000:3.0:1767320769.989565:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1279 00000100:00100000:3.0:1767320769.989573:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983153792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320769.989591:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320769.989649:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983153792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (156us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320769.989656:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1279 00000100:00100000:0.0:1767320769.989735:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983153664:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320769.989877:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983153792:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320769.989934:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320769.989939:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:1.0:1767320770.692576:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320771.524503:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320771.524513:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320771.524583:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320771.524587:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320771.524590:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320771.524598:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6014 00000100:00080000:2.0:1767320771.524602:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320771.524608:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320771.524628:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddf80 x1853163983154048/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320771.524671:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320771.524675:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320771.524677:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320771.524680:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6014 00000100:00080000:2.0:1767320771.524683:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320771.524686:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320771.524697:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de300 x1853163983154304/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320771.524707:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320771.524710:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320771.524712:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320771.524715:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6014 00000100:00080000:2.0:1767320771.524717:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320771.524721:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320771.524736:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163983154432/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320771.524755:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320771.524758:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320771.524760:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320771.524763:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6014 00000100:00080000:2.0:1767320771.524765:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320771.524768:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320771.524777:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dea00 x1853163983154816/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320771.524896:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983154688:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.524961:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320771.525009:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320771.525018:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983153920:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525041:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320771.525068:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983154560:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525092:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320771.525108:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320771.525113:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983154176:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525133:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320771.525186:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154688 00000100:00100000:3.0:1767320771.525193:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154688 00000100:00100000:3.0:1767320771.525239:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154560 00000100:00100000:3.0:1767320771.525242:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154560 00000100:00100000:3.0:1767320771.525264:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1177 00000100:00100000:3.0:1767320771.525293:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983154688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525342:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983154688:12345-0@lo:400:kworker.0 Request processed in 71us (380us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320771.525350:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1177 00000100:00100000:3.0:1767320771.525367:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1178 00000100:00100000:3.0:1767320771.525372:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604773c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983154560:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525401:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604773c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983154560:12345-0@lo:400:kworker.0 Request processed in 29us (308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320771.525406:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1178 00000100:00100000:3.0:1767320771.525610:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983153920 00000100:00100000:3.0:1767320771.525614:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983153920 00000100:00100000:3.0:1767320771.525638:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154176 00000100:00100000:3.0:1767320771.525641:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154176 00000100:00100000:3.0:1767320771.525659:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914 00000100:00100000:3.0:1767320771.525665:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983153920:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525695:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983153920:12345-0@lo:400:kworker.0 Request processed in 30us (652us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320771.525700:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914 00000100:00100000:3.0:1767320771.525711:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1915 00000100:00100000:3.0:1767320771.525717:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983154176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525741:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983154176:12345-0@lo:400:kworker.0 Request processed in 24us (607us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320771.525746:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1915 00010000:00080000:3.0:1767320771.525778:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ded80 x1853163983154176/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320787 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320771.525795:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983154176:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525805:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983154560:0@lo:400:kworker.0 00010000:00080000:3.0:1767320771.525831:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dd180 x1853163983153920/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320787 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320771.525847:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983153920:0@lo:400:kworker.0 00000100:00100000:3.0:1767320771.525855:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983154688:0@lo:400:kworker.0 00000100:00080000:2.0:1767320771.525976:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320771.525981:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320771.525983:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320771.526103:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983154048:0@lo:38:kworker.0 00000100:00100000:0.0:1767320771.526136:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320771.526173:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983154304:0@lo:38:kworker.0 00000100:00100000:0.0:1767320771.526196:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320771.526223:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983154432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320771.526299:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320771.526318:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983154816:0@lo:38:kworker.0 00000100:00100000:0.0:1767320771.526339:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320771.526369:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154048 00000100:00100000:0.0:1767320771.526373:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154048 00000100:00100000:0.0:1767320771.526390:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154304 00000100:00100000:0.0:1767320771.526392:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154304 00000100:00100000:0.0:1767320771.526399:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154816 00000100:00100000:0.0:1767320771.526401:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154816 00000100:00100000:0.0:1767320771.526409:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2533 00000100:00100000:0.0:1767320771.526413:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154048:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320771.526430:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320771.526461:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154048:12345-0@lo:38:kworker.0 Request processed in 45us (324us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320771.526467:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2533 00000100:00100000:0.0:1767320771.526479:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2534 00000100:00100000:0.0:1767320771.526482:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154304:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320771.526488:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320771.526504:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154304:12345-0@lo:38:kworker.0 Request processed in 21us (308us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320771.526507:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2534 00000100:00100000:0.0:1767320771.526513:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2535 00000100:00100000:0.0:1767320771.526515:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154816:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320771.526520:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320771.526530:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983154816:12345-0@lo:38:kworker.0 Request processed in 14us (192us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320771.526533:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2535 00000100:00080000:0.0:1767320771.526548:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320771.526552:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320771.526554:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983154048:0@lo:38:kworker.0 00000100:00100000:1.0:1767320771.526777:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154432 00000100:00100000:1.0:1767320771.526781:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154432 00000100:00100000:1.0:1767320771.526802:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 658 00000100:00100000:1.0:1767320771.526808:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345013f2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983154432:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320771.526817:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320771.526848:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345013f2450 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983154432:12345-0@lo:38:kworker.0 Request processed in 40us (549us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320771.526854:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 658 00000100:00080000:0.0:1767320771.527706:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320771.527713:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320771.527717:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983154304:0@lo:38:kworker.0 00000100:00080000:0.0:1767320771.527727:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320771.527730:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320771.527732:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983154432:0@lo:38:kworker.0 00000100:00080000:0.0:1767320771.527741:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320771.527744:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320771.527746:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983154816:0@lo:38:kworker.0 00010000:00010000:3.0:1767320771.716692:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320772.742101:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320773.765694:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320774.541650:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320774.541703:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320774.541801:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700480 00000100:00100000:2.0:1767320774.541812:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700480 00000100:00100000:2.0:1767320774.541898:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700352 00000100:00100000:2.0:1767320774.541902:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700352 00000100:00100000:2.0:1767320774.541927:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1916 00000100:00100000:2.0:1767320774.541937:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915700480:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320774.542018:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915700480:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320774.542026:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1916 00000100:00100000:2.0:1767320774.542047:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2536 00000100:00100000:2.0:1767320774.542051:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915700352:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320774.542072:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320774.542130:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915700352:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (394us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320774.542136:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2536 00000100:00080000:0.0:1767320774.542359:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320774.788599:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320775.112611:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983154944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320775.112683:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320775.113021:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983154944 00000100:00100000:0.0:1767320775.113029:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983154944 00000100:00100000:0.0:1767320775.113106:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280 00000100:00100000:0.0:1767320775.113116:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983154944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320775.113143:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320775.113186:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983154944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (503us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320775.113195:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280 00000100:00080000:3.0:1767320775.113262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320775.117600:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983154944:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320775.119834:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320775.119861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983155072:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320775.119934:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320775.120063:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155072 00000100:00100000:1.0:1767320775.120072:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155072 00000100:00100000:1.0:1767320775.120171:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281 00000100:00100000:1.0:1767320775.120181:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983155072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320775.120211:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320775.120257:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983155072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 78us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320775.120265:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281 00000100:00080000:3.0:1767320775.120473:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320775.120512:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983155072:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320775.812561:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320776.646215:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320776.646226:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320776.646316:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320776.646321:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320776.646325:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320776.646333:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6020 00000100:00080000:2.0:1767320776.646338:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320776.646345:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320776.646369:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163983155328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320776.646408:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320776.646411:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320776.646413:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320776.646416:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6020 00000100:00080000:2.0:1767320776.646419:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320776.646423:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320776.646433:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dea00 x1853163983155584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320776.646442:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320776.646445:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320776.646447:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320776.646450:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6020 00000100:00080000:2.0:1767320776.646452:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320776.646455:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320776.646465:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df480 x1853163983155712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320776.646482:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320776.646486:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320776.646487:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320776.646490:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6020 00000100:00080000:2.0:1767320776.646492:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320776.646495:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320776.646516:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983156096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320776.646646:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983155968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.646713:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320776.646757:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320776.646763:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983155200:0@lo:400:kworker.0 00000100:00100000:2.0:1767320776.646782:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155968 00000100:00100000:1.0:1767320776.646788:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320776.646791:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155968 00000100:00100000:1.0:1767320776.646814:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983155840:0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.646834:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320776.646834:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1179 00000100:00100000:2.0:1767320776.646843:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983155968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.646850:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320776.646854:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983155456:0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.646870:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320776.646889:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983155968:12345-0@lo:400:kworker.0 Request processed in 48us (176us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320776.646894:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155200 00000100:00100000:2.0:1767320776.646897:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1179 00000100:00100000:1.0:1767320776.646898:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155200 00000100:00100000:2.0:1767320776.646909:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155840 00000100:00100000:2.0:1767320776.646912:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155840 00000100:00100000:1.0:1767320776.646927:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155456 00000100:00100000:2.0:1767320776.646927:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1180 00000100:00100000:1.0:1767320776.646930:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155456 00000100:00100000:2.0:1767320776.646932:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983155840:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.646949:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1917 00000100:00100000:1.0:1767320776.646955:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983155200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320776.646962:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983155840:12345-0@lo:400:kworker.0 Request processed in 30us (127us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320776.646967:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1180 00000100:00100000:1.0:1767320776.646983:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983155200:12345-0@lo:400:kworker.0 Request processed in 28us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320776.646986:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918 00000100:00100000:1.0:1767320776.646990:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1917 00000100:00100000:2.0:1767320776.646991:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983155456:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320776.647018:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983155456:12345-0@lo:400:kworker.0 Request processed in 27us (147us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320776.647024:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918 00010000:00080000:1.0:1767320776.647040:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382df100 x1853163983155200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320792 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:2.0:1767320776.647042:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dfb80 x1853163983155456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320792 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320776.647054:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983155200:0@lo:400:kworker.0 00000100:00100000:2.0:1767320776.647054:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983155456:0@lo:400:kworker.0 00000100:00100000:1.0:1767320776.647077:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983155968:0@lo:400:kworker.0 00000100:00100000:2.0:1767320776.647077:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983155840:0@lo:400:kworker.0 00000100:00080000:3.0:1767320776.647164:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320776.647232:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983155328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320776.647262:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320776.647321:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983155584:0@lo:38:kworker.0 00000100:00100000:0.0:1767320776.647341:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320776.647363:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983155712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320776.647386:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320776.647388:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155328 00000100:00100000:2.0:1767320776.647392:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155328 00000100:00100000:2.0:1767320776.647407:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155584 00000100:00100000:2.0:1767320776.647410:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155584 00000100:00100000:2.0:1767320776.647421:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2537 00000100:00100000:0.0:1767320776.647422:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983156096:0@lo:38:kworker.0 00000100:00100000:2.0:1767320776.647425:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983155328:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320776.647443:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320776.647450:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320776.647470:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983155328:12345-0@lo:38:kworker.0 Request processed in 43us (207us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320776.647475:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767320776.647476:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2537 00000100:00080000:0.0:1767320776.647479:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320776.647482:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983155328:0@lo:38:kworker.0 00000100:00100000:2.0:1767320776.647484:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156096 00000100:00100000:2.0:1767320776.647487:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156096 00000100:00100000:2.0:1767320776.647497:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2538 00000100:00100000:2.0:1767320776.647501:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983155584:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320776.647502:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2539 00000100:00100000:0.0:1767320776.647506:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983156096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320776.647510:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767320776.647514:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320776.647536:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983155584:12345-0@lo:38:kworker.0 Request processed in 34us (193us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320776.647541:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2538 00000100:00100000:2.0:1767320776.647567:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983155712 00000100:00100000:2.0:1767320776.647570:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983155712 00000100:00100000:2.0:1767320776.647588:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 659 00000100:00100000:2.0:1767320776.647592:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983155712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320776.647602:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320776.647775:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983156096:12345-0@lo:38:kworker.0 Request processed in 266us (322us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320776.647782:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2539 00000100:00100000:2.0:1767320776.650816:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983155712:12345-0@lo:38:kworker.0 Request processed in 3212us (3419us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320776.650830:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 659 00000100:00080000:1.0:1767320776.652182:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320776.652192:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320776.652197:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983155584:0@lo:38:kworker.0 00000100:00080000:1.0:1767320776.652221:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320776.652224:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320776.652227:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983155712:0@lo:38:kworker.0 00000100:00080000:1.0:1767320776.652237:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320776.652240:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320776.652243:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983156096:0@lo:38:kworker.0 00010000:00010000:1.0:1767320776.836508:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320777.861516:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320778.884671:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320779.665523:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320779.665627:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320779.665758:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700736 00000100:00100000:2.0:1767320779.665769:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700736 00000100:00100000:2.0:1767320779.665848:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700608 00000100:00100000:2.0:1767320779.665852:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700608 00000100:00100000:2.0:1767320779.665874:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1919 00000100:00100000:2.0:1767320779.665884:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915700736:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320779.665975:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915700736:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (462us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320779.665984:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1919 00000100:00100000:2.0:1767320779.666032:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2540 00000100:00100000:2.0:1767320779.666241:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915700608:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320779.666263:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320779.666333:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915700608:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 93us (705us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320779.666341:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2540 00000100:00080000:0.0:1767320779.666536:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320779.909965:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320780.229465:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320780.229498:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983156224:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320780.229578:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320780.229895:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156224 00000100:00100000:2.0:1767320780.229904:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156224 00000100:00100000:2.0:1767320780.229986:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1282 00000100:00100000:2.0:1767320780.230006:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983156224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320780.230045:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320780.230095:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983156224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (517us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320780.230103:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1282 00000100:00100000:2.0:1767320780.230152:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983156224:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320780.230231:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320780.231295:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983156352:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320780.231355:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320780.231753:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156352 00000100:00100000:1.0:1767320780.231761:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156352 00000100:00100000:1.0:1767320780.231819:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283 00000100:00100000:1.0:1767320780.231827:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983156352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320780.231850:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320780.231916:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320780.233565:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983156352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1732us (2202us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320780.233577:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283 00000100:00100000:3.0:1767320780.233955:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983156352:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320780.932772:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320781.765265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320781.765297:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320781.765349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320781.765353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320781.765357:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320781.765365:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6025 00000100:00080000:2.0:1767320781.765370:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320781.765376:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320781.765395:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dfb80 x1853163983156608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320781.765425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320781.765428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320781.765429:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320781.765433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6025 00000100:00080000:2.0:1767320781.765435:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320781.765439:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320781.765447:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd500 x1853163983156864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320781.765455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320781.765458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320781.765459:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320781.765462:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6025 00000100:00080000:2.0:1767320781.765465:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320781.765468:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320781.765479:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163983156992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320781.765498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320781.765501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320781.765503:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320781.765506:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6025 00000100:00080000:2.0:1767320781.765509:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320781.765512:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320781.765522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983157376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320781.765551:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983156736:0@lo:400:kworker.0 00000100:00100000:2.0:1767320781.765615:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320781.765672:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320781.765679:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983157120:0@lo:400:kworker.0 00000100:00100000:2.0:1767320781.765704:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320781.765763:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983156608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320781.765840:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156736 00000100:00100000:3.0:1767320781.765848:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156736 00000100:00100000:0.0:1767320781.765883:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320781.765901:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156608 00000100:00100000:3.0:1767320781.765905:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156608 00000100:00100000:0.0:1767320781.765909:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983156864:0@lo:38:kworker.0 00000100:00100000:3.0:1767320781.765921:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920 00000100:00100000:3.0:1767320781.765930:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983156736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320781.766075:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983156480:0@lo:400:kworker.0 00000100:00080000:2.0:1767320781.766150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320781.766167:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2541 00000100:00100000:2.0:1767320781.766172:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983156608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320781.766189:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320781.774421:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320781.774449:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983156736:12345-0@lo:400:kworker.0 Request processed in 8518us (8831us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320781.774465:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920 00000100:00100000:0.0:1767320781.774469:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156864 00000100:00100000:0.0:1767320781.774476:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156864 00000100:00100000:3.0:1767320781.774506:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2542 00000100:00100000:3.0:1767320781.774513:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983156864:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320781.774532:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320781.774536:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983156992:0@lo:38:kworker.0 00000100:00100000:3.0:1767320781.774572:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983156864:12345-0@lo:38:kworker.0 Request processed in 58us (159us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320781.774578:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2542 00000100:00100000:3.0:1767320781.774596:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157120 00000100:00100000:3.0:1767320781.774600:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157120 00000100:00100000:3.0:1767320781.774630:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1181 00000100:00100000:3.0:1767320781.774636:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983157120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320781.774674:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983157120:12345-0@lo:400:kworker.0 Request processed in 38us (8969us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320781.774680:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1181 00000100:00100000:3.0:1767320781.774702:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983157248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320781.774728:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320781.774758:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157248 00000100:00100000:3.0:1767320781.774762:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157248 00000100:00100000:3.0:1767320781.774784:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1182 00000100:00100000:3.0:1767320781.774790:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983157248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320781.774822:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983157248:12345-0@lo:400:kworker.0 Request processed in 33us (94us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320781.774828:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1182 00000100:00100000:0.0:1767320781.774860:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320781.774863:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983157248:0@lo:400:kworker.0 00000100:00100000:0.0:1767320781.774883:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156992 00000100:00100000:0.0:1767320781.774887:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156992 00000100:00100000:0.0:1767320781.774907:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 660 00000100:00100000:0.0:1767320781.774911:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983156992:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320781.774912:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983157120:0@lo:400:kworker.0 00010000:00080000:3.0:1767320781.774925:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dd180 x1853163983156736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320797 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00020000:0.0:1767320781.774932:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320781.774935:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983156736:0@lo:400:kworker.0 00000100:00100000:0.0:1767320781.774956:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983156992:12345-0@lo:38:kworker.0 Request processed in 43us (95us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320781.774975:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 660 00000100:00080000:0.0:1767320781.775003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320781.775008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320781.775011:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983156992:0@lo:38:kworker.0 00000100:00100000:0.0:1767320781.775018:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983157376:0@lo:38:kworker.0 00000100:00100000:0.0:1767320781.775042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320781.775079:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320781.775083:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320781.775086:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983156864:0@lo:38:kworker.0 00000100:00100000:1.0:1767320781.776357:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320781.776389:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157376 00000100:00100000:3.0:1767320781.776397:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157376 00000100:00100000:3.0:1767320781.776835:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983156480 00000100:00100000:3.0:1767320781.776840:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983156480 00000100:00100000:1.0:1767320781.776921:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1921 00000100:00100000:1.0:1767320781.776932:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983156480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320781.777006:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983156480:12345-0@lo:400:kworker.0 Request processed in 77us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320781.777014:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1921 00000100:00100000:1.0:1767320781.777035:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2543 00000100:00100000:1.0:1767320781.777040:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983157376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320781.777060:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:0.0:1767320781.777955:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382de300 x1853163983156480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320797 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320781.777980:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983156480:0@lo:400:kworker.0 00000100:00080000:3.0:1767320781.781550:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320781.783640:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983156608:12345-0@lo:38:kworker.0 Request processed in 17456us (17746us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320781.783656:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2541 00000100:00100000:1.0:1767320781.786192:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983157376:12345-0@lo:38:kworker.0 Request processed in 9140us (11138us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320781.786209:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2543 00000100:00080000:0.0:1767320781.796322:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320781.796336:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320781.796342:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983156608:0@lo:38:kworker.0 00000100:00080000:0.0:1767320781.796373:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320781.796377:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320781.796380:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983157376:0@lo:38:kworker.0 00010000:00010000:1.0:1767320781.958807:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320782.980465:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320784.005682:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320784.783798:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320784.783860:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320784.784176:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700864 00000100:00100000:2.0:1767320784.784188:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700864 00000100:00100000:2.0:1767320784.784229:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915700992 00000100:00100000:2.0:1767320784.784231:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915700992 00000100:00100000:2.0:1767320784.784302:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1922 00000100:00100000:2.0:1767320784.784311:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915700992:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320784.784397:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915700992:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (536us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320784.784405:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1922 00000100:00100000:2.0:1767320784.784423:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2544 00000100:00100000:2.0:1767320784.784427:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915700864:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320784.784447:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320784.784476:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915700864:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (689us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320784.784483:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2544 00000100:00080000:0.0:1767320784.784656:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320785.029365:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320785.349240:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983157504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320785.349353:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320785.349485:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157504 00000100:00100000:0.0:1767320785.349493:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157504 00000100:00100000:0.0:1767320785.349551:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284 00000100:00100000:0.0:1767320785.349560:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983157504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320785.349591:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320785.349639:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983157504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320785.349646:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284 00000100:00080000:1.0:1767320785.349706:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320785.349739:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983157504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320785.356549:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320785.364480:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983157632:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320785.364560:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320785.364618:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157632 00000100:00100000:3.0:1767320785.364628:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157632 00000100:00100000:3.0:1767320785.364808:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1285 00000100:00100000:3.0:1767320785.364820:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983157632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320785.364850:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320785.364918:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983157632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 101us (357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320785.364926:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1285 00000100:00080000:1.0:1767320785.365267:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320785.365702:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983157632:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320786.052549:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320786.887171:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320786.887183:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320786.887247:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320786.887252:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320786.887255:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320786.887263:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6030 00000100:00080000:2.0:1767320786.887268:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320786.887291:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320786.887314:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382df100 x1853163983157888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320786.887345:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320786.887348:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320786.887350:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320786.887354:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6030 00000100:00080000:2.0:1767320786.887356:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320786.887361:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320786.887372:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983158144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320786.887381:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320786.887386:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320786.887388:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320786.887391:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6030 00000100:00080000:2.0:1767320786.887394:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320786.887397:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320786.887411:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dfb80 x1853163983158272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320786.887433:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320786.887436:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320786.887438:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320786.887442:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6030 00000100:00080000:2.0:1767320786.887444:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320786.887447:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320786.887457:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163983158656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320786.887490:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983158400:0@lo:400:kworker.0 00000100:00100000:2.0:1767320786.887549:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320786.887612:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320786.887618:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983158016:0@lo:400:kworker.0 00000100:00100000:2.0:1767320786.887642:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320786.887673:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158016 00000100:00100000:2.0:1767320786.887679:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158016 00000100:00100000:2.0:1767320786.887725:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1923 00000100:00100000:2.0:1767320786.887735:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983158016:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320786.887781:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983158016:12345-0@lo:400:kworker.0 Request processed in 48us (138us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320786.887789:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1923 00010000:00080000:2.0:1767320786.887823:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc000 x1853163983158016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320802 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320786.887837:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983158016:0@lo:400:kworker.0 00000100:00080000:1.0:1767320786.888302:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320786.888317:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983157760:0@lo:400:kworker.0 00000100:00100000:1.0:1767320786.888351:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320786.888377:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320786.888383:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983158528:0@lo:400:kworker.0 00000100:00100000:1.0:1767320786.888404:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320786.888745:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157760 00000100:00100000:2.0:1767320786.888752:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157760 00000100:00100000:2.0:1767320786.888797:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924 00000100:00100000:2.0:1767320786.888803:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983157760:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320786.888843:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983157760:12345-0@lo:400:kworker.0 Request processed in 39us (491us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320786.888850:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924 00000100:00080000:1.0:1767320786.888914:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320786.888937:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddf80 x1853163983157760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320802 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320786.888948:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983157760:0@lo:400:kworker.0 00000100:00100000:0.0:1767320786.889031:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983157888:0@lo:38:kworker.0 00000100:00100000:0.0:1767320786.889059:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320786.889088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983158144:0@lo:38:kworker.0 00000100:00100000:0.0:1767320786.889107:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320786.889132:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983158272:0@lo:38:kworker.0 00000100:00100000:0.0:1767320786.889152:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320786.889174:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158272 00000100:00100000:0.0:1767320786.889178:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158272 00000100:00100000:0.0:1767320786.889196:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 661 00000100:00100000:0.0:1767320786.889202:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983158272:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320786.889219:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320786.889247:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983158272:12345-0@lo:38:kworker.0 Request processed in 44us (94us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320786.889253:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 661 00000100:00080000:0.0:1767320786.889307:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320786.889311:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320786.889314:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983158272:0@lo:38:kworker.0 00000100:00100000:0.0:1767320786.889320:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983158656:0@lo:38:kworker.0 00000100:00100000:0.0:1767320786.889342:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320786.889362:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983157888 00000100:00100000:0.0:1767320786.889365:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983157888 00000100:00100000:0.0:1767320786.889376:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158144 00000100:00100000:0.0:1767320786.889378:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158144 00000100:00100000:0.0:1767320786.889385:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158656 00000100:00100000:0.0:1767320786.889387:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158656 00000100:00100000:0.0:1767320786.889396:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2545 00000100:00100000:0.0:1767320786.889400:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983157888:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320786.889410:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320786.889434:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983157888:12345-0@lo:38:kworker.0 Request processed in 32us (374us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320786.889440:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2545 00000100:00100000:0.0:1767320786.889450:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2546 00000100:00100000:0.0:1767320786.889454:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d4850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983158144:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320786.889462:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320786.889482:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d4850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983158144:12345-0@lo:38:kworker.0 Request processed in 28us (374us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320786.889488:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2546 00000100:00100000:0.0:1767320786.890444:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2547 00000100:00100000:0.0:1767320786.890450:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983158656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320786.890462:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320786.890493:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983158656:12345-0@lo:38:kworker.0 Request processed in 42us (1151us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320786.890499:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2547 00000100:00080000:0.0:1767320786.890530:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320786.890533:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320786.890536:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983158656:0@lo:38:kworker.0 00000100:00080000:0.0:1767320786.890557:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320786.890561:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320786.890564:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983157888:0@lo:38:kworker.0 00000100:00080000:0.0:1767320786.890571:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320786.890575:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320786.890577:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983158144:0@lo:38:kworker.0 00000100:00100000:3.0:1767320786.891184:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158400 00000100:00100000:3.0:1767320786.891192:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158400 00000100:00100000:3.0:1767320786.891236:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158528 00000100:00100000:3.0:1767320786.891239:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158528 00000100:00100000:3.0:1767320786.891266:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1183 00000100:00100000:3.0:1767320786.891308:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983158400:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320786.891360:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983158400:12345-0@lo:400:kworker.0 Request processed in 55us (3811us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320786.891366:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1183 00000100:00100000:3.0:1767320786.891381:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1184 00000100:00100000:3.0:1767320786.891392:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983158528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320786.891423:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983158528:12345-0@lo:400:kworker.0 Request processed in 37us (3017us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320786.891428:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1184 00000100:00100000:1.0:1767320786.891519:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983158400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320786.891549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983158528:0@lo:400:kworker.0 00010000:00010000:3.0:1767320787.076499:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320788.100508:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1767320788.875697:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915701120 offset 0 length 520: 4 00000100:00100000:1.0:1767320788.880866:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320788.880931:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320788.880977:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701248 00000100:00100000:3.0:1767320788.880987:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701248 00000100:00100000:3.0:1767320788.881035:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701376 00000100:00100000:3.0:1767320788.881038:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701376 00000100:00100000:3.0:1767320788.881089:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1925 00000100:00100000:3.0:1767320788.881101:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915701376:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320788.881185:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915701376:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320788.881194:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1925 00000100:00100000:3.0:1767320788.881213:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2548 00000100:00100000:3.0:1767320788.881217:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915701248:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320788.881236:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320788.881261:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915701248:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (407us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320788.881267:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2548 00000100:00080000:1.0:1767320788.882720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320789.125219:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320790.148499:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320790.468450:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983158784:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320790.468524:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320790.468675:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320790.468689:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983158912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320790.468714:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320790.468744:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158784 00000100:00100000:0.0:1767320790.468761:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158784 00000100:00100000:0.0:1767320790.468850:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983158912 00000100:00100000:0.0:1767320790.468854:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983158912 00000100:00100000:0.0:1767320790.468887:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1286 00000100:00100000:0.0:1767320790.468895:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983158784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320790.468924:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320790.469004:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287 00000100:00100000:3.0:1767320790.469014:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983158912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320790.469031:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320790.469067:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983158912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320790.469076:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287 00000100:00100000:3.0:1767320790.469125:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983158912:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320790.469172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320790.469176:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320790.473072:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983158784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4164us (4534us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320790.473089:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1286 00000100:00100000:1.0:1767320790.478110:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983158784:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767320791.174252:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320792.004460:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.004470:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320792.004532:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.004536:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320792.004540:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320792.004547:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6035 00000100:00080000:2.0:1767320792.004552:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320792.004559:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320792.004581:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc000 x1853163983159168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320792.004615:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.004619:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320792.004621:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320792.004625:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6035 00000100:00080000:2.0:1767320792.004627:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320792.004631:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320792.004641:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de300 x1853163983159424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320792.004650:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.004653:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320792.004655:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320792.004658:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6035 00000100:00080000:2.0:1767320792.004660:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320792.004664:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320792.004674:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983159552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320792.004693:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.004696:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320792.004698:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320792.004701:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6035 00000100:00080000:2.0:1767320792.004703:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320792.004706:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320792.004715:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dfb80 x1853163983159936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320792.004810:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983159296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.004873:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320792.004941:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320792.004948:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983159680:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.004973:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320792.005004:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159296 00000100:00100000:3.0:1767320792.005011:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159296 00000100:00100000:3.0:1767320792.005061:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926 00000100:00100000:3.0:1767320792.005072:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983159296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005116:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983159296:12345-0@lo:400:kworker.0 Request processed in 46us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320792.005123:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926 00000100:00080000:3.0:1767320792.005141:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320792.005156:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983159040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005176:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320792.005198:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320792.005203:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983159808:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005222:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320792.005245:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159680 00000100:00100000:3.0:1767320792.005249:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159680 00000100:00100000:3.0:1767320792.005300:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159808 00000100:00100000:3.0:1767320792.005302:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159808 00000100:00100000:3.0:1767320792.005319:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1185 00000100:00100000:3.0:1767320792.005325:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983159680:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005351:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983159680:12345-0@lo:400:kworker.0 Request processed in 27us (377us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320792.005356:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1185 00000100:00100000:3.0:1767320792.005367:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1186 00000100:00100000:3.0:1767320792.005371:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983159808:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005394:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983159808:12345-0@lo:400:kworker.0 Request processed in 22us (171us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320792.005399:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1186 00000100:00100000:3.0:1767320792.005418:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159040 00000100:00100000:3.0:1767320792.005421:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159040 00000100:00100000:3.0:1767320792.005440:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1927 00000100:00100000:3.0:1767320792.005445:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983159040:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005473:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983159040:12345-0@lo:400:kworker.0 Request processed in 28us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320792.005479:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1927 00000100:00080000:3.0:1767320792.005492:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320792.005495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320792.005497:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320792.005521:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983159808:0@lo:400:kworker.0 00010000:00080000:3.0:1767320792.005533:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dd880 x1853163983159040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320808 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320792.005542:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983159040:0@lo:400:kworker.0 00000100:00100000:3.0:1767320792.005572:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983159680:0@lo:400:kworker.0 00010000:00080000:3.0:1767320792.005584:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dd500 x1853163983159296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320808 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320792.005592:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983159296:0@lo:400:kworker.0 00000100:00100000:0.0:1767320792.005704:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320766/real 1767320766] req@ffff9346382de680 x1853163983152512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320791 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320792.005722:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346382de680 x1853163983152512/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320791 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320792.005733:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983159168:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.005758:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.005792:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983159424:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.005813:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.005832:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159168 00000100:00100000:0.0:1767320792.005835:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159168 00000100:00100000:0.0:1767320792.005853:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159424 00000100:00100000:0.0:1767320792.005856:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159424 00000100:00100000:0.0:1767320792.005872:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2549 00000100:00100000:0.0:1767320792.005876:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983159168:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320792.005892:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.005918:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983159168:12345-0@lo:38:kworker.0 Request processed in 40us (159us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.005923:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2549 00000100:00100000:0.0:1767320792.005934:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2550 00000100:00100000:0.0:1767320792.005938:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983159424:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320792.005947:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.005968:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983159424:12345-0@lo:38:kworker.0 Request processed in 28us (155us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.005973:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2550 00000100:00080000:0.0:1767320792.006002:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320792.006007:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320792.006010:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983159424:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.006015:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983159552:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.006038:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.006060:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159552 00000100:00100000:0.0:1767320792.006063:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159552 00000100:00100000:0.0:1767320792.006082:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 662 00000100:00100000:0.0:1767320792.006086:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983159552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320792.006096:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.006120:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983159552:12345-0@lo:38:kworker.0 Request processed in 32us (81us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.006125:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 662 00000100:00080000:0.0:1767320792.006169:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320792.006174:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320792.006177:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983159552:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.006183:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983159936:0@lo:38:kworker.0 00000100:00100000:0.0:1767320792.006202:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320792.006218:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983159936 00000100:00100000:0.0:1767320792.006221:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983159936 00000100:00100000:0.0:1767320792.006236:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2551 00000100:00100000:0.0:1767320792.006240:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983159936:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320792.006250:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320792.006287:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983159936:12345-0@lo:38:kworker.0 Request processed in 45us (84us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320792.006292:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2551 00000100:00080000:0.0:1767320792.006314:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320792.006318:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320792.006321:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983159936:0@lo:38:kworker.0 00000100:00080000:0.0:1767320792.006325:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320792.007183:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320792.007190:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983152512:0@lo:250:kworker.0 00000100:00080000:0.0:1767320792.007228:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320792.007232:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320792.007234:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983159168:0@lo:38:kworker.0 00000100:00080000:2.0:1767320792.007288:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320792.007294:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320792.007297:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320792.007301:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6014 00000100:00080000:2.0:1767320792.007305:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320792.007309:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320792.007335:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dce00 x1853163983160064/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320792.007461:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983160064:0@lo:250:kworker.0 00000400:00000100:0.0:1767320792.007491:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983160064 offset 0 length 520: 4 00010000:00010000:1.0:1767320792.197312:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320793.220722:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320793.997069:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320793.997113:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320793.997207:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701504 00000100:00100000:1.0:1767320793.997218:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701504 00000100:00100000:1.0:1767320793.997263:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701632 00000100:00100000:1.0:1767320793.997267:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701632 00000100:00100000:1.0:1767320793.997355:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1928 00000100:00100000:1.0:1767320793.997366:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915701632:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320793.997444:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915701632:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320793.997453:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1928 00000100:00100000:1.0:1767320793.997472:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2552 00000100:00100000:1.0:1767320793.997476:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915701504:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320793.997496:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320793.997537:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915701504:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (478us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320793.997544:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2552 00000100:00080000:2.0:1767320793.997879:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320794.247991:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320795.268591:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320795.588667:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983160192:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320795.588756:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320795.588838:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160192 00000100:00100000:0.0:1767320795.588846:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160192 00000100:00100000:0.0:1767320795.588902:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1288 00000100:00100000:0.0:1767320795.588909:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983160192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320795.588933:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320795.588968:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983160192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (212us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320795.588976:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1288 00000100:00100000:3.0:1767320795.589062:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983160192:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320795.589142:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320795.589357:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983160320:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320795.589385:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320795.589529:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160320 00000100:00100000:2.0:1767320795.589534:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160320 00000100:00100000:2.0:1767320795.589575:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289 00000100:00100000:2.0:1767320795.589583:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983160320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320795.589601:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320795.589637:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983160320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (251us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320795.589644:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289 00000100:00080000:3.0:1767320795.589819:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320795.589847:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983160320:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320796.295858:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320797.127389:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320797.127400:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320797.127463:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320797.127468:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320797.127472:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320797.127482:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6040 00000100:00080000:0.0:1767320797.127487:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320797.127495:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320797.127519:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377800 x1853163983160576/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320797.127567:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320797.127570:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320797.127572:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320797.127576:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6040 00000100:00080000:0.0:1767320797.127578:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320797.127582:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320797.127592:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604377480 x1853163983160832/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320797.127600:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320797.127603:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320797.127615:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320797.127619:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6040 00000100:00080000:0.0:1767320797.127622:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320797.127626:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320797.127640:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374380 x1853163983160960/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320797.127662:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320797.127666:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320797.127667:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320797.127670:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6040 00000100:00080000:0.0:1767320797.127673:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320797.127703:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320797.127717:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ce00 x1853163983161344/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320797.127732:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983161216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.127740:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983160704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320797.127830:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983160448:0@lo:400:kworker.0 00000100:00100000:2.0:1767320797.127878:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983160576:0@lo:38:kworker.0 00000100:00100000:0.0:1767320797.130524:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320797.130668:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320797.130692:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983161088:0@lo:400:kworker.0 00000100:00100000:0.0:1767320797.130731:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320797.130870:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161088 00000100:00100000:0.0:1767320797.130878:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161088 00000100:00100000:0.0:1767320797.130945:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1187 00000100:00100000:0.0:1767320797.130956:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983161088:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.131181:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320797.131224:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320797.131236:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161216 00000100:00100000:3.0:1767320797.131240:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161216 00000100:00100000:3.0:1767320797.131284:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1188 00000100:00100000:3.0:1767320797.131292:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983161216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.131332:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983161216:12345-0@lo:400:kworker.0 Request processed in 41us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320797.131340:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1188 00000100:00080000:3.0:1767320797.131360:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320797.131391:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983161216:0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.131434:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160704 00000100:00100000:3.0:1767320797.131438:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160704 00000100:00100000:3.0:1767320797.131474:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1929 00000100:00100000:3.0:1767320797.131480:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983160704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.131517:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983160704:12345-0@lo:400:kworker.0 Request processed in 37us (1002us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320797.131523:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1929 00000100:00080000:3.0:1767320797.131538:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320797.131581:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604374e00 x1853163983160704/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320813 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320797.131593:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983160704:0@lo:400:kworker.0 00000100:00100000:1.0:1767320797.132788:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320797.133981:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983161088:12345-0@lo:400:kworker.0 Request processed in 3023us (3244us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320797.134002:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1187 00000100:00100000:1.0:1767320797.134046:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983161088:0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.134198:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160448 00000100:00100000:3.0:1767320797.134206:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160448 00000100:00100000:3.0:1767320797.134266:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1930 00000100:00100000:3.0:1767320797.134294:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983160448:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320797.134348:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983160448:12345-0@lo:400:kworker.0 Request processed in 55us (1562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320797.134354:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1930 00010000:00080000:1.0:1767320797.134496:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375c00 x1853163983160448/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320813 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320797.134508:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983160448:0@lo:400:kworker.0 00000100:00080000:0.0:1767320797.134744:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320797.136194:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.137728:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983160832:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.137777:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.139823:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983160960:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.139878:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320797.139899:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983161344:0@lo:38:kworker.0 00000100:00100000:2.0:1767320797.139922:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320797.141123:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160576 00000100:00100000:3.0:1767320797.141135:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160576 00000100:00100000:1.0:1767320797.141141:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160960 00000100:00100000:1.0:1767320797.141148:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160960 00000100:00100000:3.0:1767320797.141181:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983160832 00000100:00100000:3.0:1767320797.141186:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983160832 00000100:00100000:1.0:1767320797.141189:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 663 00000100:00100000:1.0:1767320797.141199:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983160960:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320797.141210:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2553 00000100:00100000:3.0:1767320797.141217:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983160576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320797.141223:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:3.0:1767320797.141231:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320797.141305:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983160576:12345-0@lo:38:kworker.0 Request processed in 87us (5115us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320797.141311:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983160960:12345-0@lo:38:kworker.0 Request processed in 109us (1430us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320797.141318:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2553 00000100:00100000:1.0:1767320797.141324:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 663 00000100:00100000:3.0:1767320797.141341:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2554 00000100:00100000:3.0:1767320797.141347:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983160832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320797.141356:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320797.141383:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983160832:12345-0@lo:38:kworker.0 Request processed in 35us (3605us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320797.141389:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2554 00000100:00080000:2.0:1767320797.141590:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.141596:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.141601:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983160576:0@lo:38:kworker.0 00000100:00080000:2.0:1767320797.141612:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.141615:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.141618:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604377480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983160832:0@lo:38:kworker.0 00000100:00080000:2.0:1767320797.141626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.141629:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.141632:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983160960:0@lo:38:kworker.0 00000100:00100000:3.0:1767320797.141763:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161344 00000100:00100000:3.0:1767320797.141768:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161344 00000100:00100000:3.0:1767320797.141784:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2555 00000100:00100000:3.0:1767320797.141789:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983161344:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320797.141799:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320797.141837:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983161344:12345-0@lo:38:kworker.0 Request processed in 47us (1914us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320797.141843:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2555 00000100:00080000:2.0:1767320797.141964:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320797.141981:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320797.141984:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983161344:0@lo:38:kworker.0 00010000:00010000:1.0:1767320797.317075:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767320798.340472:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320799.118936:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320799.119022:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320799.119125:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701888 00000100:00100000:0.0:1767320799.119136:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701888 00000100:00100000:0.0:1767320799.119206:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915701760 00000100:00100000:0.0:1767320799.119210:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915701760 00000100:00100000:0.0:1767320799.119229:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1931 00000100:00100000:0.0:1767320799.119240:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915701888:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320799.119348:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915701888:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 111us (422us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320799.119356:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1931 00000100:00100000:0.0:1767320799.119372:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2556 00000100:00100000:0.0:1767320799.119377:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915701760:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320799.119397:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320799.119420:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915701760:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (398us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320799.119426:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2556 00000100:00080000:3.0:1767320799.119785:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320799.364564:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320800.388442:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320800.708543:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320800.708797:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320800.708812:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983161600:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320800.708879:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320800.708933:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161600 00000100:00100000:2.0:1767320800.708943:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161600 00000100:00100000:2.0:1767320800.709051:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1290 00000100:00100000:2.0:1767320800.709061:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983161600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320800.709090:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320800.709137:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983161600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320800.709144:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1290 00000100:00100000:2.0:1767320800.709189:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983161600:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320800.709310:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983161472:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320800.709339:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320800.709639:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161472 00000100:00100000:1.0:1767320800.709644:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161472 00000100:00080000:3.0:1767320800.711244:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320800.711251:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320800.711431:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1291 00000100:00100000:1.0:1767320800.711445:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983161472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320800.711486:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320800.711558:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983161472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 118us (2219us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320800.711564:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1291 00000100:00100000:0.0:1767320800.711802:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983161472:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1767320801.412513:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320802.244414:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320802.244424:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320802.244487:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320802.244491:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320802.244495:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320802.244504:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6045 00000100:00080000:0.0:1767320802.244511:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320802.244518:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320802.244540:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543dc00 x1853163983161856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320802.244603:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320802.244607:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320802.244609:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320802.244613:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6045 00000100:00080000:0.0:1767320802.244615:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320802.244619:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320802.244630:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543f100 x1853163983162112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320802.244639:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320802.244642:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320802.244644:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320802.244647:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6045 00000100:00080000:0.0:1767320802.244649:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320802.244652:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320802.244661:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543df80 x1853163983162240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320802.244678:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320802.244681:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320802.244683:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320802.244686:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6045 00000100:00080000:0.0:1767320802.244689:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320802.244691:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320802.244699:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543c000 x1853163983162624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320802.244729:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983161984:0@lo:400:kworker.0 00000100:00100000:0.0:1767320802.244800:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.244833:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320802.244838:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983162368:0@lo:400:kworker.0 00000100:00100000:0.0:1767320802.244856:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320802.244881:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162368 00000100:00100000:0.0:1767320802.244887:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162368 00000100:00100000:0.0:1767320802.244932:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1189 00000100:00100000:0.0:1767320802.244940:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983162368:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320802.244980:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983162368:12345-0@lo:400:kworker.0 Request processed in 41us (122us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320802.244986:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1189 00000100:00080000:0.0:1767320802.245003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320802.245027:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983162368:0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.245201:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983162496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.245228:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320802.245246:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320802.245250:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983161728:0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.245268:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320802.245317:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162496 00000100:00100000:3.0:1767320802.245321:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162496 00000100:00100000:3.0:1767320802.245352:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1190 00000100:00100000:3.0:1767320802.245358:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983162496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320802.245388:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983162496:12345-0@lo:400:kworker.0 Request processed in 31us (159us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320802.245394:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1190 00000100:00080000:3.0:1767320802.245410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320802.245423:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983162496:0@lo:400:kworker.0 00000100:00100000:1.0:1767320802.245571:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161984 00000100:00100000:1.0:1767320802.245575:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161984 00000100:00100000:1.0:1767320802.245619:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161728 00000100:00100000:1.0:1767320802.245622:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161728 00000100:00100000:1.0:1767320802.245640:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1932 00000100:00100000:1.0:1767320802.245646:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983161984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320802.245680:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983161984:12345-0@lo:400:kworker.0 Request processed in 34us (880us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320802.245686:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1932 00000100:00100000:1.0:1767320802.245697:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1933 00000100:00100000:1.0:1767320802.245702:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983161728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320802.245722:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983161728:12345-0@lo:400:kworker.0 Request processed in 21us (453us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320802.245727:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1933 00000100:00100000:1.0:1767320802.245748:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983161856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320802.245768:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320802.245781:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983162112:0@lo:38:kworker.0 00000100:00100000:1.0:1767320802.245796:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320802.245806:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983162240:0@lo:38:kworker.0 00000100:00100000:1.0:1767320802.245824:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320802.245852:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983162624:0@lo:38:kworker.0 00000100:00100000:1.0:1767320802.245871:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320802.245901:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162240 00000100:00100000:0.0:1767320802.245905:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983161856 00000100:00100000:1.0:1767320802.245905:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162240 00000100:00100000:0.0:1767320802.245911:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983161856 00000100:00100000:1.0:1767320802.245923:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 664 00000100:00100000:1.0:1767320802.245930:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983162240:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320802.245931:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162112 00000100:00100000:0.0:1767320802.245936:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162112 00000100:00100000:0.0:1767320802.245946:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162624 00000100:00100000:0.0:1767320802.245949:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162624 00010000:00020000:1.0:1767320802.245951:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320802.245962:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2557 00000100:00100000:0.0:1767320802.245969:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983161856:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320802.245981:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320802.245984:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983162240:12345-0@lo:38:kworker.0 Request processed in 52us (157us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320802.245991:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 664 00000100:00100000:1.0:1767320802.246015:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2558 00000100:00100000:0.0:1767320802.246021:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983161856:12345-0@lo:38:kworker.0 Request processed in 50us (250us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320802.246022:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983162112:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320802.246034:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2557 00010000:00020000:1.0:1767320802.246037:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320802.246052:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2559 00000100:00100000:0.0:1767320802.246058:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983162624:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320802.246064:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983162112:12345-0@lo:38:kworker.0 Request processed in 42us (265us total) trans 0 rc -19/-19 00010000:00020000:0.0:1767320802.246071:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320802.246075:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2558 00000100:00100000:0.0:1767320802.246101:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cf800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983162624:12345-0@lo:38:kworker.0 Request processed in 41us (229us total) trans 0 rc -19/-19 00000100:00080000:3.0:1767320802.246262:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320802.246287:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543e300 x1853163983161984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320818 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320802.246301:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983161984:0@lo:400:kworker.0 00010000:00080000:3.0:1767320802.246311:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460543d500 x1853163983161728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320818 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320802.246326:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983161728:0@lo:400:kworker.0 00000100:00080000:1.0:1767320802.246483:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320802.246488:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320802.246491:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983161856:0@lo:38:kworker.0 00000100:00080000:1.0:1767320802.246498:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320802.246501:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320802.246503:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983162112:0@lo:38:kworker.0 00000100:00080000:1.0:1767320802.246510:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320802.246512:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320802.246515:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983162240:0@lo:38:kworker.0 00000100:00080000:1.0:1767320802.246522:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320802.246525:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320802.246527:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983162624:0@lo:38:kworker.0 00000100:00100000:0.0:1767320802.246554:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2559 00010000:00010000:3.0:1767320802.436467:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320803.462268:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320804.245449:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320804.245499:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320804.245536:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702016 00000100:00100000:0.0:1767320804.245573:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702016 00000100:00100000:2.0:1767320804.245584:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702144 00000100:00100000:2.0:1767320804.245590:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702144 00000100:00100000:0.0:1767320804.245623:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2560 00000100:00100000:0.0:1767320804.245634:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915702016:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767320804.245655:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1934 00010000:00020000:0.0:1767320804.245659:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320804.245668:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915702144:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320804.245723:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915702016:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (296us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320804.245730:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915702144:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 63us (228us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320804.245738:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2560 00000100:00100000:2.0:1767320804.245746:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1934 00000100:00080000:3.0:1767320804.245920:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767320804.484532:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320805.513357:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320805.828715:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320805.828741:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983162752:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320805.828810:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320805.829049:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162752 00000100:00100000:1.0:1767320805.829059:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162752 00000100:00100000:2.0:1767320805.829298:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983162880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320805.829335:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320805.829435:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983162880 00000100:00100000:0.0:1767320805.829440:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983162880 00000100:00100000:1.0:1767320805.830401:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292 00000100:00100000:1.0:1767320805.830414:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983162752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320805.830448:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320805.830540:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983162752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 128us (1730us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320805.830549:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292 00000100:00100000:2.0:1767320805.830578:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983162752:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320805.831989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320805.831997:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320805.836784:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293 00000100:00100000:0.0:1767320805.836803:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983162880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320805.836844:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320805.836926:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983162880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 130us (7590us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320805.836934:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293 00000100:00100000:2.0:1767320805.837301:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983162880:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320806.532946:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320807.366984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320807.366996:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320807.367064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320807.367068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320807.367072:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320807.367080:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6050 00000100:00080000:1.0:1767320807.367085:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320807.367092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320807.367114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35c00 x1853163983163136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320807.367162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320807.367165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320807.367167:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320807.367171:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6050 00000100:00080000:1.0:1767320807.367173:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320807.367177:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320807.367186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34000 x1853163983163392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320807.367194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320807.367197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320807.367198:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320807.367201:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6050 00000100:00080000:1.0:1767320807.367203:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320807.367207:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320807.367216:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36d80 x1853163983163520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320807.367236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320807.367239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320807.367240:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320807.367243:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6050 00000100:00080000:1.0:1767320807.367246:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320807.367249:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320807.367258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37800 x1853163983163904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320807.367323:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983163648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320807.367405:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320807.367481:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320807.367515:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983163264:0@lo:400:kworker.0 00000100:00100000:1.0:1767320807.367543:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320807.367632:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983163776:0@lo:400:kworker.0 00000100:00100000:2.0:1767320807.367662:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320807.367692:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320807.367697:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983163008:0@lo:400:kworker.0 00000100:00100000:2.0:1767320807.367732:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320807.367778:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163264 00000100:00100000:2.0:1767320807.367787:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163264 00000100:00100000:2.0:1767320807.367834:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163008 00000100:00100000:2.0:1767320807.367838:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163008 00000100:00100000:2.0:1767320807.367863:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1935 00000100:00100000:2.0:1767320807.367872:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983163264:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320807.367924:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983163264:12345-0@lo:400:kworker.0 Request processed in 52us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320807.367932:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1935 00000100:00100000:2.0:1767320807.367948:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936 00000100:00100000:2.0:1767320807.367954:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983163008:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320807.367984:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983163008:12345-0@lo:400:kworker.0 Request processed in 31us (251us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320807.367991:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936 00010000:00080000:2.0:1767320807.368026:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35180 x1853163983163008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320823 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320807.368038:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983163008:0@lo:400:kworker.0 00010000:00080000:2.0:1767320807.368061:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34700 x1853163983163264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320823 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320807.368069:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983163264:0@lo:400:kworker.0 00000100:00100000:3.0:1767320807.368375:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163648 00000100:00100000:3.0:1767320807.368380:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163648 00000100:00100000:3.0:1767320807.368421:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163776 00000100:00100000:3.0:1767320807.368424:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163776 00000100:00100000:3.0:1767320807.368448:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1191 00000100:00100000:3.0:1767320807.368455:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983163648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320807.368510:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983163648:12345-0@lo:400:kworker.0 Request processed in 56us (1105us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320807.368516:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1191 00000100:00100000:3.0:1767320807.368530:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1192 00000100:00100000:3.0:1767320807.368536:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983163776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320807.368569:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983163776:12345-0@lo:400:kworker.0 Request processed in 33us (905us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320807.368574:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1192 00000100:00100000:3.0:1767320807.368601:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983163136:0@lo:38:kworker.0 00000100:00100000:3.0:1767320807.368627:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320807.368652:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983163392:0@lo:38:kworker.0 00000100:00100000:3.0:1767320807.368672:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320807.368688:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983163520:0@lo:38:kworker.0 00000100:00100000:3.0:1767320807.368849:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320807.368872:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983163904:0@lo:38:kworker.0 00000100:00100000:3.0:1767320807.368891:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320807.368918:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163136 00000100:00100000:3.0:1767320807.368922:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163136 00000100:00100000:3.0:1767320807.368941:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163392 00000100:00100000:3.0:1767320807.368944:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163392 00000100:00100000:3.0:1767320807.368950:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163904 00000100:00100000:3.0:1767320807.368952:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163904 00000100:00100000:3.0:1767320807.368960:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2561 00000100:00100000:3.0:1767320807.368964:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163136:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320807.368985:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320807.369018:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163136:12345-0@lo:38:kworker.0 Request processed in 53us (390us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320807.369024:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2561 00000100:00100000:3.0:1767320807.369035:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2562 00000100:00100000:3.0:1767320807.369038:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163392:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320807.369047:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320807.369066:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163392:12345-0@lo:38:kworker.0 Request processed in 26us (394us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320807.369072:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2562 00000100:00100000:3.0:1767320807.369081:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2563 00000100:00100000:3.0:1767320807.369085:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320807.369093:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320807.369114:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983163904:12345-0@lo:38:kworker.0 Request processed in 28us (223us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320807.369119:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2563 00000100:00100000:3.0:1767320807.369152:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983163776:0@lo:400:kworker.0 00000100:00080000:3.0:1767320807.369175:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320807.369180:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320807.369183:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983163136:0@lo:38:kworker.0 00000100:00080000:3.0:1767320807.369190:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320807.369193:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320807.369195:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983163392:0@lo:38:kworker.0 00000100:00080000:3.0:1767320807.369205:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320807.369208:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320807.369211:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983163904:0@lo:38:kworker.0 00000100:00100000:3.0:1767320807.369238:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983163648:0@lo:400:kworker.0 00000100:00080000:3.0:1767320807.369252:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320807.369256:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320807.369258:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320807.369366:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983163520 00000100:00100000:1.0:1767320807.369371:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983163520 00000100:00100000:1.0:1767320807.369401:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 665 00000100:00100000:1.0:1767320807.369405:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983163520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320807.369415:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320807.369449:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983163520:12345-0@lo:38:kworker.0 Request processed in 41us (600us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320807.369455:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 665 00000100:00080000:1.0:1767320807.369477:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320807.369481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320807.369485:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983163520:0@lo:38:kworker.0 00010000:00010000:2.0:1767320807.556729:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320808.580722:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320809.361561:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320809.361610:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320809.361645:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702400 00000100:00100000:2.0:1767320809.361665:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702400 00000100:00100000:1.0:1767320809.361703:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702272 00000100:00100000:1.0:1767320809.361710:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702272 00000100:00080000:3.0:1767320809.361779:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320809.361911:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1937 00000100:00100000:2.0:1767320809.361931:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915702400:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320809.362021:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915702400:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (471us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320809.362029:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1937 00000100:00100000:2.0:1767320809.362045:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2564 00000100:00100000:2.0:1767320809.362049:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915702272:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320809.362067:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320809.362091:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915702272:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (482us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320809.362097:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2564 00010000:00010000:3.0:1767320809.610294:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320810.628676:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320810.949459:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320810.949482:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983164160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320810.949551:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320810.949625:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164160 00000100:00100000:3.0:1767320810.949632:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164160 00000100:00100000:2.0:1767320810.949659:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320810.949669:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983164032:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320810.949690:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1294 00000100:00100000:2.0:1767320810.949696:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320810.949703:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983164160:12345-0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320810.949726:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00002000:00000024:3.0:1767320810.949733:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320810.949787:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983164160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 89us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320810.949794:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1294 00000100:00100000:3.0:1767320810.949809:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164032 00000100:00100000:3.0:1767320810.949812:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164032 00000100:00100000:3.0:1767320810.949832:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295 00000100:00100000:3.0:1767320810.949838:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983164032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320810.949852:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320810.949877:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983164032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320810.949884:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295 00000100:00080000:2.0:1767320810.949995:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320810.950027:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983164032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320810.950055:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983164160:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320811.652587:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320812.485761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320812.485777:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320812.485834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320812.485839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320812.485843:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320812.485851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6055 00000100:00080000:1.0:1767320812.485856:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320812.485863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320812.485880:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983164288:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.485885:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34380 x1853163983164416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320812.485944:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983164416:0@lo:38:kworker.0 00000100:00100000:2.0:1767320812.486015:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320812.486115:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164288 00000100:00100000:3.0:1767320812.486124:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164288 00000100:00100000:0.0:1767320812.486161:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983164544:0@lo:400:kworker.0 00000100:00080000:1.0:1767320812.486195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767320812.486199:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938 00000100:00080000:1.0:1767320812.486202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320812.486205:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320812.486210:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6055 00000100:00080000:1.0:1767320812.486213:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320812.486215:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983164288:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767320812.486220:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320812.486236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a37100 x1853163983164672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320812.491665:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320812.492042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320812.492149:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983164288:12345-0@lo:400:kworker.0 Request processed in 5937us (6132us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320812.492163:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938 00000100:00100000:3.0:1767320812.492188:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164416 00000100:00100000:3.0:1767320812.492196:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164416 00000100:00100000:3.0:1767320812.492235:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2565 00000100:00100000:3.0:1767320812.492241:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983164416:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320812.492263:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320812.493622:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.494097:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983164672:0@lo:38:kworker.0 00000100:00100000:0.0:1767320812.494109:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164544 00000100:00100000:0.0:1767320812.494128:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164544 00000100:00100000:2.0:1767320812.494155:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320812.494226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320812.494232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320812.494236:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320812.494243:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6055 00000100:00080000:1.0:1767320812.494247:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320812.494254:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983164416:12345-0@lo:38:kworker.0 Request processed in 2008us (2212us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767320812.494256:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320812.494267:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2565 00000100:00080000:2.0:1767320812.494410:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.494416:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.494420:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983164416:0@lo:38:kworker.0 00000100:00100000:0.0:1767320812.497991:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164672 00000100:00100000:0.0:1767320812.498008:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164672 00000100:00100000:0.0:1767320812.498054:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1939 00000100:00100000:0.0:1767320812.498072:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983164544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320812.500942:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2566 00000100:00100000:2.0:1767320812.500956:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983164672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320812.500980:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320812.501077:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983164672:12345-0@lo:38:kworker.0 Request processed in 121us (6921us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320812.501084:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2566 00000100:00080000:2.0:1767320812.501143:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320812.501149:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320812.501154:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a37100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983164672:0@lo:38:kworker.0 00010000:00080000:2.0:1767320812.501318:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36d80 x1853163983164288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320828 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320812.501332:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983164288:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.501789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a36680 x1853163983164800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320812.501823:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983164800:0@lo:38:kworker.0 00000100:00100000:1.0:1767320812.502106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320812.502187:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983164928:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.502213:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320812.502347:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164928 00000100:00100000:1.0:1767320812.502355:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164928 00000100:00100000:1.0:1767320812.502409:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1193 00000100:00100000:1.0:1767320812.502418:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983164928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.502633:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983164928:12345-0@lo:400:kworker.0 Request processed in 46us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320812.502644:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1193 00000100:00100000:1.0:1767320812.502685:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983164928:0@lo:400:kworker.0 00000100:00100000:3.0:1767320812.502762:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983164800 00000100:00100000:3.0:1767320812.502767:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983164800 00000100:00100000:3.0:1767320812.502802:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 666 00000100:00100000:3.0:1767320812.502808:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983164800:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320812.502827:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320812.502880:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983164800:12345-0@lo:38:kworker.0 Request processed in 71us (774us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320812.502890:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 666 00000100:00080000:3.0:1767320812.502923:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320812.502928:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320812.502931:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983164800:0@lo:38:kworker.0 00000100:00080000:2.0:1767320812.503037:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320812.503042:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320812.504136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320812.504144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320812.504147:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320812.504154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6055 00000100:00080000:1.0:1767320812.504158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320812.504164:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320812.504182:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320812.504195:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983165056:0@lo:400:kworker.0 00000100:00100000:0.0:1767320812.504250:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983164544:12345-0@lo:400:kworker.0 Request processed in 6184us (10632us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320812.504261:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1939 00000100:00100000:1.0:1767320812.504301:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320812.504341:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165056 00000100:00100000:1.0:1767320812.504347:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165056 00010000:00080000:2.0:1767320812.504364:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a34a80 x1853163983164544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320828 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320812.504379:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983164544:0@lo:400:kworker.0 00000100:00100000:1.0:1767320812.504390:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1194 00000100:00100000:1.0:1767320812.504410:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983165056:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320812.504411:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320812.504478:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983165056:12345-0@lo:400:kworker.0 Request processed in 57us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320812.504485:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1194 00000100:00100000:1.0:1767320812.504523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35500 x1853163983165184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320812.504570:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983165056:0@lo:400:kworker.0 00000100:00100000:3.0:1767320812.505137:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983165184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320812.505174:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320812.505235:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165184 00000100:00100000:2.0:1767320812.505241:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165184 00000100:00100000:2.0:1767320812.505287:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2567 00000100:00100000:2.0:1767320812.505293:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983165184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320812.505311:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320812.505357:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983165184:12345-0@lo:38:kworker.0 Request processed in 62us (181us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320812.505364:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2567 00000100:00080000:1.0:1767320812.505382:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320812.505388:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320812.505401:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983165184:0@lo:38:kworker.0 00010000:00010000:3.0:1767320812.676449:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320813.700548:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320814.476909:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320814.476948:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915702528 offset 0 length 520: 4 00000100:00100000:3.0:1767320814.476971:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320814.478005:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702784 00000100:00100000:0.0:1767320814.478018:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702784 00000100:00100000:0.0:1767320814.478090:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702656 00000100:00100000:0.0:1767320814.478093:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702656 00000100:00100000:0.0:1767320814.478112:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1940 00000100:00100000:0.0:1767320814.478121:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915702784:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320814.478205:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915702784:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (1305us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320814.478213:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1940 00000100:00100000:0.0:1767320814.478230:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2568 00000100:00100000:0.0:1767320814.478235:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915702656:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320814.478255:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320814.478302:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915702656:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 65us (1329us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320814.478309:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2568 00000100:00080000:3.0:1767320814.478936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320814.729800:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320815.748816:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320816.069781:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983165312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320816.069854:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320816.070042:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983165440:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320816.070073:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320816.070128:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165312 00000100:00100000:0.0:1767320816.070138:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165312 00000100:00100000:0.0:1767320816.070209:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165440 00000100:00100000:0.0:1767320816.070213:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165440 00000100:00100000:3.0:1767320816.070227:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296 00000100:00080000:2.0:1767320816.070266:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320816.070289:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320816.070529:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983165312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320816.070562:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320816.070622:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983165312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 385us (767us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320816.070717:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983165312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320816.074795:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1297 00000100:00100000:0.0:1767320816.074821:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983165440:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320816.074856:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320816.074948:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983165440:12345-0@lo:13:osp-pre-0-1.0 Request processed in 140us (4876us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320816.074956:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1297 00000100:00100000:1.0:1767320816.075176:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983165440:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320816.078652:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296 00010000:00010000:2.0:1767320816.781541:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320817.608113:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320792/real 1767320792] req@ffff9346382dce00 x1853163983160064/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320817 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320817.608150:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346382dce00 x1853163983160064/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320817 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767320817.608171:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320817.608180:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767320817.608185:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983160064:0@lo:250:kworker.0 00000100:00080000:2.0:1767320817.608661:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320817.608669:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320817.608673:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320817.608678:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6040 00000100:00080000:2.0:1767320817.608682:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320817.608689:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320817.608725:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dce00 x1853163983165568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320817.608800:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320817.608804:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320817.608806:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320817.608810:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6061 00000100:00080000:2.0:1767320817.608813:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320817.608816:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320817.608830:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8380 x1853163983165824/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320817.608860:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320817.608863:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320817.608865:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320817.608869:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6061 00000100:00080000:2.0:1767320817.608871:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320817.608875:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320817.608885:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fad80 x1853163983166080/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320817.608894:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320817.608898:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320817.608899:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320817.608903:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6061 00000100:00080000:2.0:1767320817.608905:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320817.608909:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320817.608930:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163983166208/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320817.608950:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320817.608953:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320817.608955:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320817.608958:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6061 00000100:00080000:2.0:1767320817.608960:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320817.608963:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320817.608971:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb480 x1853163983166592/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320817.608998:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983165952:0@lo:400:kworker.0 00000100:00100000:2.0:1767320817.609060:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.609116:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320817.609122:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983166336:0@lo:400:kworker.0 00000100:00100000:2.0:1767320817.609146:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.609332:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983165696:0@lo:400:kworker.0 00000100:00100000:3.0:1767320817.609362:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.609413:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320817.609419:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983166464:0@lo:400:kworker.0 00000100:00100000:3.0:1767320817.609440:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.609467:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165952 00000100:00100000:3.0:1767320817.609475:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165952 00000100:00100000:3.0:1767320817.609522:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165696 00000100:00100000:3.0:1767320817.609525:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165696 00000100:00100000:3.0:1767320817.609547:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941 00000100:00100000:3.0:1767320817.609567:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983165952:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.609586:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166336 00000100:00100000:1.0:1767320817.609593:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166336 00000100:00100000:3.0:1767320817.609624:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983165952:12345-0@lo:400:kworker.0 Request processed in 68us (562us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320817.609631:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941 00000100:00100000:1.0:1767320817.609640:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166464 00000100:00100000:1.0:1767320817.609644:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166464 00000100:00100000:3.0:1767320817.609665:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942 00000100:00100000:1.0:1767320817.609670:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1195 00000100:00100000:3.0:1767320817.609676:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983165696:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.609683:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983166336:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320817.609720:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983165696:12345-0@lo:400:kworker.0 Request processed in 46us (356us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320817.609726:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942 00000100:00100000:1.0:1767320817.609734:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983166336:12345-0@lo:400:kworker.0 Request processed in 55us (584us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320817.609741:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1195 00000100:00100000:3.0:1767320817.609750:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1196 00000100:00100000:3.0:1767320817.609759:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983166464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.609768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983165568:0@lo:250:kworker.0 00000100:00100000:3.0:1767320817.609791:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983166464:12345-0@lo:400:kworker.0 Request processed in 33us (349us total) trans 0 rc 0/0 00000400:00000100:1.0:1767320817.609797:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983165568 offset 0 length 520: 4 00000100:00100000:3.0:1767320817.609802:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1196 00000100:00080000:3.0:1767320817.609825:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320817.609829:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983165824:0@lo:38:kworker.0 00000100:00100000:1.0:1767320817.609857:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.609868:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983166464:0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.609879:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983166080:0@lo:38:kworker.0 00010000:00080000:3.0:1767320817.609885:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8000 x1853163983165696/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320833 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320817.609896:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983165696:0@lo:400:kworker.0 00000100:00100000:1.0:1767320817.609905:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320817.609923:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983166208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320817.609924:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983165824 00000100:00100000:3.0:1767320817.609929:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983165824 00000100:00100000:1.0:1767320817.609953:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320817.609953:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166080 00000100:00100000:3.0:1767320817.609959:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166080 00000100:00100000:3.0:1767320817.609975:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2569 00000100:00100000:3.0:1767320817.609982:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983165824:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320817.609990:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983166592:0@lo:38:kworker.0 00010000:00020000:3.0:1767320817.609999:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320817.610013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320817.610140:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166592 00000100:00100000:2.0:1767320817.610145:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166592 00000100:00100000:2.0:1767320817.610160:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2570 00000100:00100000:2.0:1767320817.610165:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983166080:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320817.610176:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320817.610246:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2571 00000100:00100000:0.0:1767320817.610253:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983166592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320817.610262:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320817.617224:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983165824:12345-0@lo:38:kworker.0 Request processed in 7229us (7352us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320817.617241:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2569 00000100:00100000:3.0:1767320817.617320:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166208 00000100:00100000:3.0:1767320817.617330:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166208 00000100:00100000:3.0:1767320817.617375:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 667 00000100:00100000:3.0:1767320817.617382:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983166208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320817.617406:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320817.617455:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983166208:12345-0@lo:38:kworker.0 Request processed in 72us (7503us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320817.617461:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 667 00000100:00100000:3.0:1767320817.617548:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983166336:0@lo:400:kworker.0 00010000:00080000:3.0:1767320817.618239:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8e00 x1853163983165952/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320833 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320817.618251:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983165952:0@lo:400:kworker.0 00000100:00100000:2.0:1767320817.618548:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983166080:12345-0@lo:38:kworker.0 Request processed in 8378us (8641us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320817.618576:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2570 00000100:00100000:0.0:1767320817.627136:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983166592:12345-0@lo:38:kworker.0 Request processed in 16871us (17111us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320817.627153:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2571 00000100:00080000:1.0:1767320817.628638:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320817.628648:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320817.628652:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983166592:0@lo:38:kworker.0 00000100:00080000:1.0:1767320817.628685:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320817.628688:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320817.628691:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983165824:0@lo:38:kworker.0 00000100:00080000:1.0:1767320817.628701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320817.628705:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320817.628707:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983166080:0@lo:38:kworker.0 00000100:00080000:1.0:1767320817.628715:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320817.628718:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320817.628720:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983166208:0@lo:38:kworker.0 00010000:00010000:3.0:1767320817.799429:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320818.823366:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320819.597857:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320819.597931:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915702912 00000100:00100000:0.0:1767320819.597943:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915702912 00000100:00100000:0.0:1767320819.597998:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2572 00000100:00100000:0.0:1767320819.598007:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915702912:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320819.598029:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320819.598104:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915702912:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 97us (257us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320819.598111:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2572 00000100:00100000:3.0:1767320819.598680:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320819.598953:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703040 00000100:00100000:1.0:1767320819.598959:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703040 00000100:00080000:2.0:1767320819.599012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320819.599017:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1943 00000100:00100000:1.0:1767320819.599028:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703040:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320819.599085:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703040:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 60us (407us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320819.599093:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1943 00010000:00010000:2.0:1767320819.844564:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320820.868582:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320821.188709:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320821.188731:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983166720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320821.188810:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320821.188936:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166720 00000100:00100000:0.0:1767320821.188947:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166720 00000100:00100000:0.0:1767320821.189017:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1298 00000100:00100000:0.0:1767320821.189027:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983166720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320821.189056:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320821.189103:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983166720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (293us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320821.189110:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1298 00000100:00080000:2.0:1767320821.189201:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320821.189247:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983166720:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320821.189428:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320821.193346:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983166848:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320821.193437:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320821.193618:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166848 00000100:00100000:0.0:1767320821.193627:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166848 00000100:00100000:0.0:1767320821.193691:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1299 00000100:00100000:0.0:1767320821.193701:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983166848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320821.193732:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320821.193787:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983166848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320821.193794:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1299 00000100:00080000:2.0:1767320821.193942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320821.194011:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983166848:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320821.892479:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320822.724664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320822.724674:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320822.724736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320822.724740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320822.724745:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320822.724753:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320822.724757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320822.724764:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320822.724785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323b80 x1853163983167104/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320822.724853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983167104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320822.724923:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320822.725002:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167104 00000100:00100000:1.0:1767320822.725010:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167104 00000100:00100000:1.0:1767320822.725047:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2573 00000100:00100000:1.0:1767320822.725055:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983167104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320822.725075:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320822.725154:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320822.725165:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983166976:0@lo:400:kworker.0 00000100:00100000:2.0:1767320822.725194:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320822.725236:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983166976 00000100:00100000:2.0:1767320822.725241:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983166976 00000100:00100000:2.0:1767320822.725305:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:2.0:1767320822.725313:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983166976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320822.725356:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983166976:12345-0@lo:400:kworker.0 Request processed in 44us (161us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320822.725362:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00000100:00080000:2.0:1767320822.725386:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320822.725411:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322a00 x1853163983166976/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320838 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320822.725424:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983166976:0@lo:400:kworker.0 00000100:00080000:3.0:1767320822.728366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320822.728372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320822.728376:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320822.728383:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320822.728387:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320822.728393:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320822.728412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323100 x1853163983167360/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320822.728482:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983167232:0@lo:400:kworker.0 00000100:00100000:2.0:1767320822.728541:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320822.728594:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167232 00000100:00100000:2.0:1767320822.728600:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167232 00000100:00100000:2.0:1767320822.728649:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1945 00000100:00100000:2.0:1767320822.728656:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983167232:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320822.728697:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983167232:12345-0@lo:400:kworker.0 Request processed in 41us (156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320822.728704:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1945 00000100:00080000:2.0:1767320822.728724:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320822.728746:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320700 x1853163983167232/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320838 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320822.728758:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983167232:0@lo:400:kworker.0 00000100:00100000:0.0:1767320822.730467:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983167360:0@lo:38:kworker.0 00000100:00100000:0.0:1767320822.730644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320822.730730:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983167104:12345-0@lo:38:kworker.0 Request processed in 5673us (5805us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320822.730741:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2573 00000100:00100000:1.0:1767320822.730779:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167360 00000100:00100000:1.0:1767320822.730786:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167360 00000100:00100000:1.0:1767320822.730811:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2574 00000100:00100000:1.0:1767320822.730816:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983167360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320822.730836:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320822.730864:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983167360:12345-0@lo:38:kworker.0 Request processed in 48us (221us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320822.730870:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2574 00000100:00080000:3.0:1767320822.735444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320822.735453:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320822.735457:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320822.735464:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6066 00000100:00080000:3.0:1767320822.735468:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320822.735475:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320822.735670:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9880 x1853163983167488/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320822.735728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320822.735731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320822.735733:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320822.735736:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6066 00000100:00080000:2.0:1767320822.735738:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320822.735741:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320822.735751:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb100 x1853163983167872/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320822.735781:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983167744:0@lo:400:kworker.0 00000100:00100000:2.0:1767320822.735846:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320822.736048:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167744 00000100:00100000:3.0:1767320822.736057:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167744 00000100:00100000:3.0:1767320822.736113:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1197 00000100:00100000:3.0:1767320822.736122:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983167744:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320822.736181:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983167744:12345-0@lo:400:kworker.0 Request processed in 60us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320822.736188:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1197 00000100:00100000:3.0:1767320822.736218:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320822.736227:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983167616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320822.736249:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320822.736268:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167616 00000100:00100000:3.0:1767320822.736295:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167616 00000100:00100000:3.0:1767320822.736315:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1198 00000100:00100000:3.0:1767320822.736320:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983167616:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320822.736350:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983167616:12345-0@lo:400:kworker.0 Request processed in 29us (100us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320822.736355:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1198 00000100:00100000:3.0:1767320822.736388:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983167616:0@lo:400:kworker.0 00000100:00100000:3.0:1767320822.736412:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983167744:0@lo:400:kworker.0 00000100:00080000:0.0:1767320822.739790:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320822.739803:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320822.739809:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983167104:0@lo:38:kworker.0 00000100:00100000:0.0:1767320822.739849:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983167488:0@lo:38:kworker.0 00000100:00100000:0.0:1767320822.739917:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320822.739963:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983167872:0@lo:38:kworker.0 00000100:00100000:0.0:1767320822.739988:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320822.740111:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167488 00000100:00100000:3.0:1767320822.740119:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167488 00000100:00100000:3.0:1767320822.740179:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 668 00000100:00100000:3.0:1767320822.740186:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983167488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320822.740219:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320822.740263:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983167488:12345-0@lo:38:kworker.0 Request processed in 76us (346us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320822.740295:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 668 00000100:00080000:0.0:1767320822.741753:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320822.741762:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320822.741766:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983167360:0@lo:38:kworker.0 00000100:00080000:0.0:1767320822.741794:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320822.741797:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320822.741801:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983167488:0@lo:38:kworker.0 00000100:00100000:2.0:1767320822.744691:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983167872 00000100:00100000:2.0:1767320822.744708:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983167872 00000100:00100000:2.0:1767320822.744842:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2575 00000100:00100000:2.0:1767320822.744851:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983167872:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320822.744877:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320822.744968:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983167872:12345-0@lo:38:kworker.0 Request processed in 116us (4978us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320822.744974:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2575 00000100:00080000:1.0:1767320822.745057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320822.745064:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320822.745068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983167872:0@lo:38:kworker.0 00010000:00010000:2.0:1767320822.922493:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320823.940657:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320824.716744:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320824.716793:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320824.716881:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703168 00000100:00100000:2.0:1767320824.716892:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703168 00000100:00100000:2.0:1767320824.716939:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703296 00000100:00100000:2.0:1767320824.716943:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703296 00000100:00100000:2.0:1767320824.716998:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1946 00000100:00100000:2.0:1767320824.717010:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703296:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320824.717104:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703296:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (309us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320824.717113:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1946 00000100:00100000:2.0:1767320824.717146:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2576 00000100:00100000:2.0:1767320824.717151:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915703168:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320824.717203:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320824.717234:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915703168:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 81us (502us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320824.717240:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2576 00000100:00080000:3.0:1767320824.717479:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320824.968190:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320825.988549:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320826.309388:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320826.309409:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983168000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320826.309480:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320826.309762:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168000 00000100:00100000:0.0:1767320826.309771:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168000 00000100:00100000:0.0:1767320826.309840:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1300 00000100:00100000:0.0:1767320826.309850:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983168000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320826.309879:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320826.309937:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983168000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (456us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320826.309945:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1300 00000100:00080000:1.0:1767320826.310076:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320826.310114:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983168000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320826.310526:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320826.310547:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983168128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320826.310578:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320826.310788:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168128 00000100:00100000:2.0:1767320826.310793:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168128 00000100:00100000:2.0:1767320826.310840:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1301 00000100:00100000:2.0:1767320826.310848:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983168128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320826.310866:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320826.310902:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983168128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (323us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320826.310908:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1301 00000100:00080000:1.0:1767320826.311069:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320826.311095:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604321f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983168128:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320827.012700:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320827.845391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320827.845401:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320827.845485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320827.845489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320827.845493:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320827.845501:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320827.845505:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320827.845511:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320827.845536:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a34a80 x1853163983168384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320827.845570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320827.845573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320827.845575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320827.845579:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320827.845581:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320827.845585:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320827.845597:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198a680 x1853163983168640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320827.845605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320827.845608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320827.845610:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320827.845613:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320827.845615:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320827.845618:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320827.845629:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988a80 x1853163983168768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320827.845647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320827.845650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320827.845651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320827.845655:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6071 00000100:00080000:1.0:1767320827.845657:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320827.845660:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320827.845668:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b800 x1853163983169152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320827.845692:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983168512:0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.845752:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320827.845786:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320827.845792:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983168896:0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.845817:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320827.845855:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168512 00000100:00100000:1.0:1767320827.845861:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168512 00000100:00100000:1.0:1767320827.845914:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1947 00000100:00100000:1.0:1767320827.845924:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983168512:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.845974:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983168512:12345-0@lo:400:kworker.0 Request processed in 53us (222us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320827.845981:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1947 00000100:00080000:1.0:1767320827.846002:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320827.846017:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983168384:0@lo:38:kworker.0 00000100:00100000:1.0:1767320827.846039:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320827.846054:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983168640:0@lo:38:kworker.0 00000100:00100000:1.0:1767320827.846071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320827.846084:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983168768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320827.846105:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320827.846146:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983169152:0@lo:38:kworker.0 00000100:00100000:3.0:1767320827.846146:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983169024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.846168:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320827.846172:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320827.846189:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320827.846193:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168768 00000100:00100000:3.0:1767320827.846194:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983168256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.846196:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168768 00000100:00100000:1.0:1767320827.846215:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 669 00000100:00100000:1.0:1767320827.846220:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983168768:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320827.846227:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767320827.846237:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320827.846250:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168896 00000100:00100000:3.0:1767320827.846254:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168896 00000100:00100000:1.0:1767320827.846267:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983168768:12345-0@lo:38:kworker.0 Request processed in 46us (161us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320827.846297:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169024 00000100:00100000:1.0:1767320827.846299:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 669 00000100:00100000:3.0:1767320827.846300:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169024 00000100:00100000:1.0:1767320827.846316:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168384 00000100:00100000:3.0:1767320827.846318:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767320827.846319:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168384 00000100:00100000:3.0:1767320827.846324:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983168896:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320827.846332:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168640 00000100:00100000:1.0:1767320827.846335:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168640 00000100:00100000:1.0:1767320827.846342:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169152 00000100:00100000:1.0:1767320827.846344:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169152 00000100:00100000:1.0:1767320827.846351:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983168256 00000100:00100000:1.0:1767320827.846353:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983168256 00000100:00100000:3.0:1767320827.846355:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983168896:12345-0@lo:400:kworker.0 Request processed in 32us (538us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320827.846361:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1199 00000100:00100000:1.0:1767320827.846372:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948 00000100:00100000:3.0:1767320827.846372:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1200 00000100:00100000:1.0:1767320827.846378:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983168256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320827.846378:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983169024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320827.846407:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983169024:12345-0@lo:400:kworker.0 Request processed in 29us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320827.846413:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1200 00000100:00100000:3.0:1767320827.846442:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2577 00000100:00100000:3.0:1767320827.846447:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983168384:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320827.846457:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320827.846493:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983168384:12345-0@lo:38:kworker.0 Request processed in 44us (452us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320827.846499:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2577 00000100:00100000:3.0:1767320827.846512:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2578 00000100:00100000:3.0:1767320827.846515:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983168640:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320827.846524:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320827.846545:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983168640:12345-0@lo:38:kworker.0 Request processed in 28us (473us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320827.846551:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2578 00000100:00100000:3.0:1767320827.846561:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2579 00000100:00100000:3.0:1767320827.846565:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983169152:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320827.846573:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320827.846591:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983169152:12345-0@lo:38:kworker.0 Request processed in 26us (423us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320827.846597:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2579 00000100:00100000:1.0:1767320827.848624:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983168256:12345-0@lo:400:kworker.0 Request processed in 2236us (2385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320827.848637:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948 00000100:00080000:1.0:1767320827.848677:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320827.848717:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983168896:0@lo:400:kworker.0 00010000:00080000:1.0:1767320827.848733:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36680 x1853163983168512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320843 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320827.848744:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983168512:0@lo:400:kworker.0 00000100:00080000:1.0:1767320827.848789:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320827.848794:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320827.848797:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983168384:0@lo:38:kworker.0 00000100:00080000:1.0:1767320827.848806:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320827.848809:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320827.848811:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983168640:0@lo:38:kworker.0 00000100:00080000:1.0:1767320827.848820:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320827.848823:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320827.848825:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983168768:0@lo:38:kworker.0 00000100:00080000:1.0:1767320827.848834:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320827.848838:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320827.848840:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983169152:0@lo:38:kworker.0 00010000:00080000:3.0:1767320827.851659:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a36d80 x1853163983168256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320843 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320827.851678:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a36d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983168256:0@lo:400:kworker.0 00000100:00100000:3.0:1767320827.851694:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983169024:0@lo:400:kworker.0 00010000:00010000:3.0:1767320828.038894:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320829.070391:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320829.838606:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320829.839181:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320829.839472:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703424 00000100:00100000:3.0:1767320829.839483:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703424 00000100:00100000:3.0:1767320829.839534:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703552 00000100:00100000:3.0:1767320829.839537:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703552 00000100:00100000:0.0:1767320829.839567:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2580 00000100:00080000:1.0:1767320829.839619:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320829.839743:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915703424:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320829.839769:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320829.840031:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1949 00000100:00100000:3.0:1767320829.840041:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703552:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320829.840147:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915703552:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 108us (967us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320829.840155:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1949 00000100:00100000:1.0:1767320829.840561:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915703424:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 819us (1963us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320829.840569:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2580 00010000:00010000:3.0:1767320830.084974:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320831.109593:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320831.429461:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320831.429480:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983169280:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320831.429542:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320831.429605:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169280 00000100:00100000:0.0:1767320831.429612:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169280 00000100:00100000:0.0:1767320831.429664:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1302 00000100:00100000:0.0:1767320831.429671:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983169280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320831.429697:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320831.429734:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983169280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320831.429741:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1302 00000100:00080000:0.0:1767320831.429758:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320831.429788:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983169280:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320831.430157:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320831.430166:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983169408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320831.430190:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320831.430290:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169408 00000100:00100000:3.0:1767320831.430295:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169408 00000100:00100000:3.0:1767320831.430334:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1303 00000100:00100000:3.0:1767320831.430340:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983169408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320831.430357:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320831.430386:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983169408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 47us (196us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320831.430393:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1303 00000100:00080000:0.0:1767320831.430468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320831.430489:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983169408:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767320832.132836:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320832.964581:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320832.964605:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320832.964669:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320832.964673:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320832.964678:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320832.964686:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6076 00000100:00080000:2.0:1767320832.964691:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320832.964697:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320832.964718:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb480 x1853163983169664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320832.964765:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320832.964769:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320832.964770:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320832.964774:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6076 00000100:00080000:2.0:1767320832.964776:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320832.964780:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320832.964794:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163983169920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320832.964802:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320832.964805:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320832.964807:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320832.964809:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6076 00000100:00080000:2.0:1767320832.964812:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320832.964815:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320832.964826:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983170048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320832.964847:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320832.964851:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320832.964852:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320832.964855:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6076 00000100:00080000:2.0:1767320832.964857:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320832.964861:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320832.964871:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9880 x1853163983170432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320832.964897:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983169792:0@lo:400:kworker.0 00000100:00100000:2.0:1767320832.964965:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320832.965055:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320832.965061:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983170176:0@lo:400:kworker.0 00000100:00100000:2.0:1767320832.965087:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320832.965119:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170176 00000100:00100000:2.0:1767320832.965126:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170176 00000100:00100000:2.0:1767320832.965182:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1201 00000100:00100000:2.0:1767320832.965193:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983170176:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320832.965243:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983170176:12345-0@lo:400:kworker.0 Request processed in 51us (154us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320832.965250:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1201 00000100:00100000:2.0:1767320832.965342:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983170176:0@lo:400:kworker.0 00000100:00100000:3.0:1767320832.965436:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983169664:0@lo:38:kworker.0 00000100:00100000:3.0:1767320832.965469:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320832.965502:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983169920:0@lo:38:kworker.0 00000100:00100000:3.0:1767320832.965521:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320832.965540:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983170048:0@lo:38:kworker.0 00000100:00100000:3.0:1767320832.965562:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320832.965583:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983170432:0@lo:38:kworker.0 00000100:00100000:3.0:1767320832.965603:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320832.965629:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169792 00000100:00100000:3.0:1767320832.965634:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169792 00000100:00100000:3.0:1767320832.965668:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169664 00000100:00100000:3.0:1767320832.965671:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169664 00000100:00100000:3.0:1767320832.965680:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169920 00000100:00100000:3.0:1767320832.965683:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169920 00000100:00100000:3.0:1767320832.965689:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170432 00000100:00100000:3.0:1767320832.965691:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170432 00000100:00100000:3.0:1767320832.965701:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950 00000100:00100000:3.0:1767320832.965707:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983169792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320832.965739:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983169792:12345-0@lo:400:kworker.0 Request processed in 33us (775us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320832.965746:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950 00000100:00100000:3.0:1767320832.965760:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2581 00000100:00100000:3.0:1767320832.965763:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983169664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320832.965779:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320832.965809:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983169664:12345-0@lo:38:kworker.0 Request processed in 43us (338us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320832.965814:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2581 00000100:00100000:3.0:1767320832.965826:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2582 00000100:00100000:3.0:1767320832.965830:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983169920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320832.965839:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320832.965862:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983169920:12345-0@lo:38:kworker.0 Request processed in 30us (340us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320832.965867:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2582 00000100:00100000:3.0:1767320832.965877:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2583 00000100:00100000:3.0:1767320832.965881:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983170432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320832.965889:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320832.965910:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983170432:12345-0@lo:38:kworker.0 Request processed in 28us (308us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320832.965916:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2583 00000100:00100000:3.0:1767320832.965932:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170048 00000100:00100000:3.0:1767320832.965935:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170048 00000100:00100000:3.0:1767320832.965952:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 670 00000100:00100000:3.0:1767320832.965957:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983170048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320832.965966:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320832.965989:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983170048:12345-0@lo:38:kworker.0 Request processed in 31us (426us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320832.965994:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 670 00000100:00080000:3.0:1767320832.966009:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320832.966044:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320832.966048:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320832.966052:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983169664:0@lo:38:kworker.0 00000100:00080000:3.0:1767320832.966060:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320832.966063:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320832.966066:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983169920:0@lo:38:kworker.0 00000100:00080000:3.0:1767320832.966073:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320832.966076:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320832.966078:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983170048:0@lo:38:kworker.0 00000100:00080000:3.0:1767320832.966086:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320832.966088:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320832.966091:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983170432:0@lo:38:kworker.0 00000100:00100000:0.0:1767320832.966262:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983169536:0@lo:400:kworker.0 00000100:00100000:0.0:1767320832.966316:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320832.966346:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320832.966353:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983170304:0@lo:400:kworker.0 00000100:00100000:0.0:1767320832.966375:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320832.966499:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983169536 00000100:00100000:3.0:1767320832.966504:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983169536 00000100:00100000:3.0:1767320832.966546:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1951 00000100:00100000:3.0:1767320832.966552:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983169536:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320832.966589:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983169536:12345-0@lo:400:kworker.0 Request processed in 38us (272us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320832.966596:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1951 00010000:00080000:3.0:1767320832.966623:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8380 x1853163983169536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320848 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320832.966633:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983169536:0@lo:400:kworker.0 00000100:00080000:0.0:1767320832.966884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320832.966945:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170304 00000100:00100000:2.0:1767320832.966949:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170304 00000100:00100000:2.0:1767320832.966985:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1202 00000100:00100000:2.0:1767320832.966991:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983170304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320832.967172:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983170304:12345-0@lo:400:kworker.0 Request processed in 171us (786us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320832.967179:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1202 00010000:00080000:2.0:1767320832.967210:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8e00 x1853163983169792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320848 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320832.967222:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983169792:0@lo:400:kworker.0 00000100:00100000:0.0:1767320832.975359:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983170304:0@lo:400:kworker.0 00010000:00010000:3.0:1767320833.161136:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320834.190414:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320834.956948:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320834.957059:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320834.957200:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703808 00000100:00100000:1.0:1767320834.957209:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703808 00000100:00100000:2.0:1767320834.957291:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915703680 00000100:00100000:2.0:1767320834.957294:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915703680 00000100:00100000:2.0:1767320834.957330:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2584 00000100:00100000:2.0:1767320834.957335:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915703680:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320834.957364:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320834.957430:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915703680:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (368us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320834.957438:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2584 00000100:00100000:1.0:1767320834.957551:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1952 00000100:00100000:1.0:1767320834.957560:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915703808:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320834.957622:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915703808:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 65us (684us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320834.957627:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1952 00000100:00080000:0.0:1767320834.959213:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320835.205442:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320836.231239:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320836.548585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983170560:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320836.548667:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320836.548803:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170560 00000100:00100000:2.0:1767320836.548812:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170560 00000100:00100000:2.0:1767320836.548883:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1304 00000100:00100000:2.0:1767320836.548894:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983170560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320836.548928:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767320836.549176:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320836.550486:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983170688:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320836.550549:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320836.550871:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170688 00000100:00100000:3.0:1767320836.550879:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170688 00000100:00100000:3.0:1767320836.550934:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1305 00000100:00100000:3.0:1767320836.550959:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983170688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320836.550986:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320836.551034:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983170688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (484us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320836.551041:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1305 00000100:00100000:3.0:1767320836.551093:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983170688:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320836.551504:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320836.551736:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983170560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2843us (3067us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320836.551745:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1304 00000100:00100000:1.0:1767320836.552144:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983170560:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320837.252471:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320838.086738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320838.086748:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320838.086858:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983170816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320838.086934:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.087052:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170816 00000100:00100000:2.0:1767320838.087060:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170816 00000100:00100000:2.0:1767320838.087123:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1953 00000100:00100000:2.0:1767320838.087133:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983170816:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320838.087199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320838.087203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320838.087207:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320838.087213:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6081 00000100:00080000:3.0:1767320838.087217:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320838.087223:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320838.087245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604323b80 x1853163983170944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320838.087297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320838.087301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320838.087303:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320838.087306:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6081 00000100:00080000:3.0:1767320838.087309:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320838.087312:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320838.087324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163983171200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320838.087334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320838.087338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320838.087339:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320838.087343:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6081 00000100:00080000:3.0:1767320838.087345:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320838.087348:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320838.087358:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163983171328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320838.087429:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983170944:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.087474:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.087502:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983171200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.087524:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.087541:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983171328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.087564:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.087957:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171328 00000100:00100000:0.0:1767320838.087963:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171328 00000100:00100000:0.0:1767320838.087994:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 671 00000100:00100000:0.0:1767320838.088000:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983171328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320838.088018:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320838.088056:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983171328:12345-0@lo:38:kworker.0 Request processed in 55us (490us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320838.088063:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 671 00000100:00080000:0.0:1767320838.088100:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320838.088104:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.088108:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983171328:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.088147:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983170944 00000100:00100000:0.0:1767320838.088150:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983170944 00000100:00100000:0.0:1767320838.088166:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171200 00000100:00100000:0.0:1767320838.088168:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171200 00000100:00100000:0.0:1767320838.088178:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2585 00000100:00100000:0.0:1767320838.088182:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983170944:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320838.088193:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320838.088222:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983170944:12345-0@lo:38:kworker.0 Request processed in 38us (748us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320838.088228:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2585 00000100:00100000:0.0:1767320838.088238:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2586 00000100:00100000:0.0:1767320838.088241:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983171200:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320838.088248:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320838.088266:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983171200:12345-0@lo:38:kworker.0 Request processed in 24us (742us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320838.088294:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2586 00000100:00100000:0.0:1767320838.088313:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983171456:0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088342:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.088365:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320838.088371:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983171072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088392:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.088418:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983171584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088440:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320838.088548:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171072 00000100:00100000:0.0:1767320838.088552:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171072 00000100:00100000:0.0:1767320838.088576:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1954 00000100:00100000:0.0:1767320838.088582:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983171072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088618:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983171072:12345-0@lo:400:kworker.0 Request processed in 36us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320838.088623:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1954 00010000:00080000:0.0:1767320838.088648:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3b80 x1853163983171072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320854 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320838.088660:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983171072:0@lo:400:kworker.0 00000100:00080000:0.0:1767320838.088682:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320838.088685:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.088701:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604323b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983170944:0@lo:38:kworker.0 00000100:00080000:0.0:1767320838.088712:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320838.088715:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320838.088718:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983171200:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.088754:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171456 00000100:00100000:0.0:1767320838.088758:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171456 00000100:00100000:0.0:1767320838.088783:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171584 00000100:00100000:0.0:1767320838.088786:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171584 00000100:00100000:0.0:1767320838.088803:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1203 00000100:00100000:0.0:1767320838.088809:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983171456:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088842:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983171456:12345-0@lo:400:kworker.0 Request processed in 33us (499us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320838.088848:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1203 00000100:00100000:0.0:1767320838.088859:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1204 00000100:00100000:0.0:1767320838.088865:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983171584:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088895:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983171584:12345-0@lo:400:kworker.0 Request processed in 30us (454us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320838.088900:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1204 00000100:00100000:0.0:1767320838.088952:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983171584:0@lo:400:kworker.0 00000100:00100000:0.0:1767320838.088974:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983171456:0@lo:400:kworker.0 00000100:00080000:0.0:1767320838.089012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320838.089035:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320838.089037:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320838.092390:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983170816:12345-0@lo:400:kworker.0 Request processed in 5248us (5444us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320838.092405:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1953 00000100:00080000:3.0:1767320838.096178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320838.096189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320838.096193:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320838.096201:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6081 00000100:00080000:3.0:1767320838.096207:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320838.096214:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320838.096255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1500 x1853163983171712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320838.096408:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983171712:0@lo:38:kworker.0 00000100:00100000:0.0:1767320838.096475:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320838.096658:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171712 00000100:00100000:2.0:1767320838.096667:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171712 00000100:00100000:2.0:1767320838.096712:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2587 00000100:00100000:2.0:1767320838.096719:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983171712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320838.096739:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320838.096794:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983171712:12345-0@lo:38:kworker.0 Request processed in 74us (318us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320838.096801:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2587 00000100:00080000:3.0:1767320838.097232:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320838.097238:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320838.097242:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983171712:0@lo:38:kworker.0 00010000:00080000:1.0:1767320838.097600:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604322300 x1853163983170816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320854 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320838.097614:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604322300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983170816:0@lo:400:kworker.0 00010000:00010000:2.0:1767320838.276504:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320839.053331:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320839.053450:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704192 00000100:00100000:1.0:1767320839.053462:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704192 00000100:00100000:1.0:1767320839.053539:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1955 00000100:00100000:1.0:1767320839.053550:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915704192:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320839.053630:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915704192:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (312us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320839.053638:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1955 00000100:00080000:0.0:1767320839.054293:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767320839.054716:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915703936 offset 0 length 520: 4 00000100:00100000:3.0:1767320839.054751:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320839.055194:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704064 00000100:00100000:2.0:1767320839.055202:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704064 00000100:00100000:2.0:1767320839.055245:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2588 00000100:00100000:2.0:1767320839.055252:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915704064:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320839.055311:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320839.055371:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915704064:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 118us (619us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320839.055378:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2588 00010000:00010000:3.0:1767320839.300816:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320840.324523:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320841.352197:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320841.669031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983171840:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320841.669102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320841.669327:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320841.669340:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983171968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320841.669381:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320841.669729:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171840 00000100:00100000:1.0:1767320841.669738:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171840 00000100:00080000:0.0:1767320841.669799:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320841.673576:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983171968 00000100:00100000:3.0:1767320841.673595:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983171968 00000100:00100000:1.0:1767320841.675635:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1306 00000100:00100000:1.0:1767320841.675659:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983171840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320841.675696:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320841.675779:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983171840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 133us (6675us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320841.675787:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1306 00000100:00080000:0.0:1767320841.678319:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320841.678653:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983171840:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320841.678774:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1307 00000100:00100000:3.0:1767320841.678792:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983171968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320841.678821:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320841.678908:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983171968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (9525us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320841.678915:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1307 00000100:00100000:3.0:1767320841.678966:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983171968:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320842.184005:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320817/real 1767320817] req@ffff9346382dce00 x1853163983165568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320842.184044:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346382dce00 x1853163983165568/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320842.184066:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.184096:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320842.184100:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983165568:0@lo:250:kworker.0 00000100:00080000:2.0:1767320842.184225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320842.184231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320842.184235:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320842.184239:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6066 00000100:00080000:2.0:1767320842.184244:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320842.184250:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320842.184316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8e00 x1853163983172096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320842.184390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320842.184394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320842.184396:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320842.184399:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6086 00000100:00080000:2.0:1767320842.184402:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320842.184406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320842.184419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8380 x1853163983172352/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320842.184444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320842.184448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320842.184450:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320842.184454:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6086 00000100:00080000:2.0:1767320842.184462:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320842.184466:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320842.184478:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983172608/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320842.184486:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320842.184489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320842.184491:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320842.184494:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6086 00000100:00080000:2.0:1767320842.184497:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320842.184500:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320842.184511:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163983172736/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320842.184520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320842.184524:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320842.184525:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320842.184528:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6086 00000100:00080000:2.0:1767320842.184530:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320842.184533:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320842.184541:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fb480 x1853163983172864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320842.184566:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983172480:0@lo:400:kworker.0 00000100:00100000:2.0:1767320842.184631:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320842.184754:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320842.184765:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983172224:0@lo:400:kworker.0 00000100:00100000:1.0:1767320842.184793:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320842.184835:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172480 00000100:00100000:1.0:1767320842.184851:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172480 00000100:00100000:1.0:1767320842.184910:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172224 00000100:00100000:1.0:1767320842.184913:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172224 00000100:00100000:1.0:1767320842.184938:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1956 00000100:00100000:1.0:1767320842.184947:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983172480:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320842.185010:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983172480:12345-0@lo:400:kworker.0 Request processed in 64us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320842.185019:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1956 00000100:00100000:1.0:1767320842.185036:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1957 00000100:00100000:1.0:1767320842.185042:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983172224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320842.185070:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983172224:12345-0@lo:400:kworker.0 Request processed in 28us (276us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320842.185075:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1957 00010000:00080000:1.0:1767320842.185111:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fb100 x1853163983172224/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320858 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320842.185122:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983172224:0@lo:400:kworker.0 00000100:00080000:1.0:1767320842.185138:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320842.185154:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f9880 x1853163983172480/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320858 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320842.185162:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983172480:0@lo:400:kworker.0 00000100:00100000:3.0:1767320842.185414:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983172096:0@lo:250:kworker.0 00000400:00000100:3.0:1767320842.185451:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983172096 offset 0 length 520: 4 00000100:00100000:3.0:1767320842.185471:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983172352:0@lo:38:kworker.0 00000100:00100000:3.0:1767320842.185492:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.185517:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983172608:0@lo:38:kworker.0 00000100:00100000:3.0:1767320842.185536:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.185557:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983172736:0@lo:38:kworker.0 00000100:00100000:3.0:1767320842.185579:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320842.185596:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983172864:0@lo:38:kworker.0 00000100:00100000:3.0:1767320842.185615:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320842.185676:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172352 00000100:00100000:1.0:1767320842.185681:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172352 00000100:00100000:1.0:1767320842.185709:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172608 00000100:00100000:1.0:1767320842.185712:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172608 00000100:00100000:1.0:1767320842.185723:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172864 00000100:00100000:1.0:1767320842.185725:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172864 00000100:00100000:1.0:1767320842.185734:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2589 00000100:00100000:1.0:1767320842.185739:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172352:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320842.185756:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320842.185787:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172352:12345-0@lo:38:kworker.0 Request processed in 46us (294us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320842.185793:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2589 00000100:00100000:1.0:1767320842.185804:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2590 00000100:00100000:1.0:1767320842.185807:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172608:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320842.185816:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320842.185835:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172608:12345-0@lo:38:kworker.0 Request processed in 27us (299us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320842.185840:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2590 00000100:00100000:1.0:1767320842.185864:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2591 00000100:00100000:1.0:1767320842.185868:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172864:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320842.185877:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320842.185900:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983172864:12345-0@lo:38:kworker.0 Request processed in 31us (284us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320842.185905:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2591 00000100:00080000:3.0:1767320842.185977:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.185983:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320842.185986:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983172352:0@lo:38:kworker.0 00000100:00080000:3.0:1767320842.185993:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.185996:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320842.185999:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983172608:0@lo:38:kworker.0 00000100:00080000:3.0:1767320842.186006:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320842.186009:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320842.186012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983172864:0@lo:38:kworker.0 00000100:00100000:0.0:1767320842.186099:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172736 00000100:00100000:0.0:1767320842.186104:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172736 00000100:00100000:0.0:1767320842.186135:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 672 00000100:00100000:0.0:1767320842.186140:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983172736:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320842.186151:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320842.186185:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983172736:12345-0@lo:38:kworker.0 Request processed in 45us (605us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320842.186190:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 672 00000100:00080000:0.0:1767320842.186217:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320842.186220:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320842.186223:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983172736:0@lo:38:kworker.0 00010000:00010000:3.0:1767320842.372855:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320843.396481:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320844.171930:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320844.172019:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320844.172106:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704320 00000100:00100000:1.0:1767320844.172119:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704320 00000100:00100000:1.0:1767320844.172169:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704448 00000100:00100000:1.0:1767320844.172173:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704448 00000100:00100000:1.0:1767320844.172222:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1958 00000100:00100000:1.0:1767320844.172232:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915704448:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320844.172346:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915704448:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (325us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320844.172355:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1958 00000100:00100000:1.0:1767320844.172381:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2592 00000100:00100000:1.0:1767320844.172386:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915704320:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320844.172407:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320844.172434:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915704320:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (518us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320844.172440:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2592 00000100:00080000:0.0:1767320844.172468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320844.420635:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320845.444689:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320846.469606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320846.788494:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320846.788506:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320846.788515:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983172992:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320846.788527:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983173120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320846.795495:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320846.797190:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320846.807314:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983172992 00000100:00100000:3.0:1767320846.807331:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983172992 00000100:00100000:3.0:1767320846.807504:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173120 00000100:00100000:3.0:1767320846.807509:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173120 00000100:00100000:2.0:1767320846.807588:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1308 00000100:00100000:2.0:1767320846.807601:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983172992:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320846.807636:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320846.807732:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983172992:12345-0@lo:13:osp-pre-0-1.0 Request processed in 134us (12243us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320846.807739:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1308 00000100:00100000:2.0:1767320846.807826:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983172992:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320846.817344:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320846.817355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320846.818734:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1309 00000100:00100000:3.0:1767320846.818750:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983173120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320846.818814:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320846.818917:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983173120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 164us (21722us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320846.818925:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1309 00000100:00100000:2.0:1767320846.821225:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983173120:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320847.300402:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320847.300414:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320847.300470:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320847.300475:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320847.300479:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320847.300487:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6090 00000100:00080000:2.0:1767320847.300493:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320847.300499:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320847.300532:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f800 x1853163983173376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320847.300586:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320847.300590:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320847.300592:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320847.300596:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6090 00000100:00080000:2.0:1767320847.300598:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320847.300602:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320847.300613:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163983173632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320847.300622:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320847.300625:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320847.300627:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320847.300630:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6090 00000100:00080000:2.0:1767320847.300633:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320847.300637:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320847.300648:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f100 x1853163983173760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320847.300657:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320847.300660:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320847.300662:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320847.300664:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6090 00000100:00080000:2.0:1767320847.300667:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320847.300670:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320847.300680:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396e300 x1853163983173888/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320847.300710:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983173504:0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.300785:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320847.300840:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173504 00000100:00100000:2.0:1767320847.300848:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173504 00000100:00100000:2.0:1767320847.300901:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959 00000100:00100000:2.0:1767320847.300911:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983173504:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320847.300976:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983173504:12345-0@lo:400:kworker.0 Request processed in 67us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320847.300984:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959 00010000:00080000:2.0:1767320847.301028:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ce00 x1853163983173504/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320863 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320847.301041:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983173504:0@lo:400:kworker.0 00000100:00100000:1.0:1767320847.301574:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983173376:0@lo:38:kworker.0 00000100:00100000:1.0:1767320847.301610:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320847.301641:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983173632:0@lo:38:kworker.0 00000100:00100000:1.0:1767320847.301662:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320847.301679:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983173760:0@lo:38:kworker.0 00000100:00100000:1.0:1767320847.301702:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320847.301718:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983173888:0@lo:38:kworker.0 00000100:00100000:1.0:1767320847.301737:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320847.301773:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320847.301787:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173376 00000100:00100000:1.0:1767320847.301791:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173376 00000100:00100000:1.0:1767320847.301810:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173632 00000100:00100000:1.0:1767320847.301812:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173632 00000100:00100000:1.0:1767320847.301822:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173888 00000100:00100000:1.0:1767320847.301824:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173888 00000100:00100000:1.0:1767320847.301833:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2593 00000100:00100000:1.0:1767320847.301838:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173376:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320847.301855:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320847.301887:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173376:12345-0@lo:38:kworker.0 Request processed in 48us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320847.301893:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2593 00000100:00100000:1.0:1767320847.301905:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2594 00000100:00100000:1.0:1767320847.301909:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173632:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320847.301927:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320847.301951:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173632:12345-0@lo:38:kworker.0 Request processed in 40us (288us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320847.301957:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2594 00000100:00100000:1.0:1767320847.301967:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2595 00000100:00100000:1.0:1767320847.301970:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320847.301978:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320847.301999:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631989500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983173888:12345-0@lo:38:kworker.0 Request processed in 27us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320847.302004:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2595 00000100:00080000:1.0:1767320847.302038:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320847.302043:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320847.302047:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983173376:0@lo:38:kworker.0 00000100:00080000:1.0:1767320847.302056:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320847.302059:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320847.302062:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983173632:0@lo:38:kworker.0 00000100:00080000:1.0:1767320847.302069:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320847.302071:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320847.302074:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983173888:0@lo:38:kworker.0 00000100:00100000:0.0:1767320847.302375:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983173248:0@lo:400:kworker.0 00000100:00100000:0.0:1767320847.302405:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320847.302438:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173248 00000100:00100000:0.0:1767320847.302442:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173248 00000100:00100000:0.0:1767320847.302474:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1960 00000100:00100000:0.0:1767320847.302480:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983173248:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320847.302514:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983173248:12345-0@lo:400:kworker.0 Request processed in 34us (108us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320847.302520:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1960 00000100:00080000:0.0:1767320847.302537:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320847.302564:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c380 x1853163983173248/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320863 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320847.302576:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983173248:0@lo:400:kworker.0 00000100:00100000:3.0:1767320847.303134:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983173760 00000100:00100000:3.0:1767320847.303141:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983173760 00000100:00100000:3.0:1767320847.303173:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 673 00000100:00100000:3.0:1767320847.303179:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983173760:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320847.303194:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320847.303244:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983173760:12345-0@lo:38:kworker.0 Request processed in 64us (1541us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320847.303251:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 673 00000100:00080000:1.0:1767320847.303473:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320847.303477:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320847.303481:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983173760:0@lo:38:kworker.0 00010000:00010000:2.0:1767320847.492515:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320848.522284:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320849.292918:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320849.292971:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320849.293058:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704576 00000100:00100000:2.0:1767320849.293069:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704704 00000100:00100000:1.0:1767320849.293070:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704576 00000100:00100000:2.0:1767320849.293077:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704704 00000100:00100000:1.0:1767320849.293134:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1961 00000100:00100000:1.0:1767320849.293146:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915704704:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320849.293148:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2596 00000100:00100000:2.0:1767320849.293159:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915704576:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320849.293183:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320849.293303:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320849.293375:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604322300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915704704:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 231us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320849.293385:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1961 00000100:00100000:2.0:1767320849.295860:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915704576:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 2688us (2943us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320849.295876:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2596 00010000:00010000:3.0:1767320849.540499:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320850.565585:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320851.588748:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320851.908680:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983174016:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320851.908765:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320851.908875:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174016 00000100:00100000:2.0:1767320851.908883:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174016 00000100:00100000:2.0:1767320851.908958:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1310 00000100:00100000:2.0:1767320851.908967:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324c3c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983174016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320851.908997:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320851.909046:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324c3c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983174016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (280us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320851.909063:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1310 00000100:00100000:2.0:1767320851.909327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983174016:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320851.909651:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320851.909869:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320851.909881:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983174144:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320851.909934:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320851.910079:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174144 00000100:00100000:3.0:1767320851.910085:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174144 00000100:00100000:3.0:1767320851.910135:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1311 00000100:00100000:3.0:1767320851.910143:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983174144:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320851.910165:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320851.910209:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983174144:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320851.910216:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1311 00000100:00100000:3.0:1767320851.910251:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983174144:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320851.910383:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320852.421707:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320852.421716:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767320852.421812:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767320852.421876:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320852.421880:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320852.421883:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320852.421891:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6095 00000100:00080000:2.0:1767320852.421896:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320852.421903:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320852.421927:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163983174400/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320852.421968:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320852.421972:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320852.421973:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320852.421977:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6095 00000100:00080000:2.0:1767320852.421980:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320852.421983:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320852.421994:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c000 x1853163983174656/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320852.422002:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320852.422005:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320852.422007:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320852.422011:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6095 00000100:00080000:2.0:1767320852.422013:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320852.422016:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320852.422027:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f480 x1853163983174784/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320852.422036:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320852.422040:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320852.422042:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320852.422045:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6095 00000100:00080000:2.0:1767320852.422047:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320852.422050:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320852.422059:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163983174912/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320852.422093:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983174528:0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.422593:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320852.422645:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174528 00000100:00100000:2.0:1767320852.422663:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174528 00000100:00100000:2.0:1767320852.422721:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1962 00000100:00100000:2.0:1767320852.422731:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983174528:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320852.422783:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983174528:12345-0@lo:400:kworker.0 Request processed in 54us (201us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320852.422791:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1962 00000100:00080000:2.0:1767320852.422811:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320852.422842:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ca80 x1853163983174528/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320868 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320852.422860:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983174528:0@lo:400:kworker.0 00000100:00100000:1.0:1767320852.423246:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983174400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320852.423299:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.423327:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983174656:0@lo:38:kworker.0 00000100:00100000:1.0:1767320852.423358:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.423383:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983174784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320852.423408:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.423443:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983174912:0@lo:38:kworker.0 00000100:00100000:1.0:1767320852.423467:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320852.423507:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174784 00000100:00100000:1.0:1767320852.423511:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174784 00000100:00100000:1.0:1767320852.423533:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 674 00000100:00100000:1.0:1767320852.423539:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983174784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320852.423558:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320852.423606:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983174784:12345-0@lo:38:kworker.0 Request processed in 66us (197us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320852.423612:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 674 00000100:00080000:1.0:1767320852.423640:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320852.423645:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320852.423648:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983174784:0@lo:38:kworker.0 00000100:00100000:3.0:1767320852.423871:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174400 00000100:00100000:3.0:1767320852.423876:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174400 00000100:00100000:3.0:1767320852.423906:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174656 00000100:00100000:3.0:1767320852.423909:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174656 00000100:00100000:3.0:1767320852.423924:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174912 00000100:00100000:3.0:1767320852.423927:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174912 00000100:00100000:3.0:1767320852.423936:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2597 00000100:00100000:3.0:1767320852.423940:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320852.423951:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320852.423983:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174400:12345-0@lo:38:kworker.0 Request processed in 41us (683us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320852.423989:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2597 00000100:00100000:3.0:1767320852.424002:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2598 00000100:00100000:3.0:1767320852.424006:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174656:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320852.424016:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320852.424038:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174656:12345-0@lo:38:kworker.0 Request processed in 30us (680us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320852.424044:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2598 00000100:00100000:3.0:1767320852.424055:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2599 00000100:00100000:3.0:1767320852.424059:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174912:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320852.424068:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320852.424103:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983174912:12345-0@lo:38:kworker.0 Request processed in 42us (635us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320852.424117:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2599 00000100:00100000:0.0:1767320852.424298:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983174272:0@lo:400:kworker.0 00000100:00100000:0.0:1767320852.424330:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320852.424481:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983174272 00000100:00100000:3.0:1767320852.424486:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983174272 00000100:00100000:3.0:1767320852.424533:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1963 00000100:00100000:3.0:1767320852.424539:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983174272:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320852.424580:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983174272:12345-0@lo:400:kworker.0 Request processed in 41us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320852.424600:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1963 00000100:00080000:0.0:1767320852.424791:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320852.424816:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396ce00 x1853163983174272/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320868 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320852.424832:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983174272:0@lo:400:kworker.0 00000100:00080000:1.0:1767320852.424908:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320852.424913:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320852.424916:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983174400:0@lo:38:kworker.0 00000100:00080000:1.0:1767320852.424924:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320852.424927:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320852.424930:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983174656:0@lo:38:kworker.0 00000100:00080000:1.0:1767320852.424938:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320852.424941:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320852.424943:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983174912:0@lo:38:kworker.0 00010000:00010000:3.0:1767320852.612811:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320853.636500:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320854.412713:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320854.412761:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320854.412880:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704960 00000100:00100000:2.0:1767320854.412889:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704960 00000100:00100000:2.0:1767320854.413303:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915704832 00000100:00100000:2.0:1767320854.413307:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915704832 00000100:00080000:0.0:1767320854.413340:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320854.418606:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1964 00000100:00100000:1.0:1767320854.418644:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915704960:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320854.418752:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915704960:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 133us (6050us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320854.418760:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1964 00000100:00100000:1.0:1767320854.418785:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2600 00000100:00100000:1.0:1767320854.418789:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915704832:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320854.418809:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320854.418836:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915704832:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (6076us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320854.418842:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2600 00010000:00010000:3.0:1767320854.660724:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320855.684455:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320856.708620:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320857.029054:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983175040:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320857.029135:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320857.029257:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175040 00000100:00100000:1.0:1767320857.029266:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175040 00000100:00100000:1.0:1767320857.029345:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1312 00000100:00100000:1.0:1767320857.029354:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983175040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320857.029381:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320857.029850:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983175168:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320857.029882:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.029917:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175168 00000100:00100000:2.0:1767320857.029921:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175168 00000100:00100000:2.0:1767320857.029949:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1313 00000100:00100000:2.0:1767320857.029955:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983175168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320857.029973:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320857.030011:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983175168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 56us (128us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320857.030018:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1313 00000100:00100000:2.0:1767320857.030064:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983175168:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320857.030093:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320857.030097:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320857.030234:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983175040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 881us (1098us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320857.030241:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1312 00000100:00100000:0.0:1767320857.030365:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983175040:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320857.540876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320857.541154:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320857.541240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320857.541245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320857.541249:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320857.541259:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6100 00000100:00080000:3.0:1767320857.541266:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320857.541320:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1767320857.541343:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320857.541352:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983175296:0@lo:400:kworker.0 00000100:00100000:3.0:1767320857.541373:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2a00 x1853163983175424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320857.541420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320857.541425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320857.541427:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320857.541432:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6100 00000100:00080000:3.0:1767320857.541436:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767320857.541446:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320857.541447:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320857.541469:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c1f80 x1853163983175680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320857.541481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320857.541484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320857.541487:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320857.541492:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6100 00000100:00080000:3.0:1767320857.541495:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320857.541500:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320857.541523:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2d80 x1853163983175808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320857.541576:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983175552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.541608:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.541632:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320857.541639:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983175936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.541669:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.541700:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175296 00000100:00100000:2.0:1767320857.541708:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175296 00000100:00100000:2.0:1767320857.541758:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175552 00000100:00100000:2.0:1767320857.541761:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175552 00000100:00100000:2.0:1767320857.541782:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1965 00000100:00100000:2.0:1767320857.541790:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983175296:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.541832:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983175296:12345-0@lo:400:kworker.0 Request processed in 43us (392us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320857.541840:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1965 00000100:00100000:2.0:1767320857.541853:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1966 00000100:00100000:2.0:1767320857.541858:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983175552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.541882:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983175552:12345-0@lo:400:kworker.0 Request processed in 24us (275us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320857.541887:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1966 00000100:00080000:2.0:1767320857.541910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320857.541936:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0e00 x1853163983175552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320857.541969:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983175552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.541994:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983176064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.542014:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767320857.542032:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c0a80 x1853163983175296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320857.542040:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983175296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320857.542121:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983175424:0@lo:38:kworker.0 00000100:00100000:1.0:1767320857.542149:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320857.542180:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983175680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320857.542200:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320857.542218:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983175808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320857.542241:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320857.542292:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175808 00000100:00100000:1.0:1767320857.542295:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175808 00000100:00100000:1.0:1767320857.542317:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 675 00000100:00100000:1.0:1767320857.542323:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983175808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320857.542341:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320857.542372:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983175808:12345-0@lo:38:kworker.0 Request processed in 48us (130us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320857.542378:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 675 00000100:00100000:1.0:1767320857.542393:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175424 00000100:00100000:1.0:1767320857.542396:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175424 00000100:00100000:1.0:1767320857.542410:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175680 00000100:00100000:1.0:1767320857.542413:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175680 00000100:00100000:1.0:1767320857.542426:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2601 00000100:00100000:1.0:1767320857.542430:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983175424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320857.542439:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320857.542463:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983175424:12345-0@lo:38:kworker.0 Request processed in 31us (313us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320857.542468:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2601 00000100:00100000:1.0:1767320857.542479:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2602 00000100:00100000:1.0:1767320857.542483:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983175680:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320857.542491:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320857.542512:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983175680:12345-0@lo:38:kworker.0 Request processed in 28us (311us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320857.542517:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2602 00000100:00080000:1.0:1767320857.542548:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320857.542553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320857.542556:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983175424:0@lo:38:kworker.0 00000100:00080000:1.0:1767320857.542578:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320857.542582:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320857.542584:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983175680:0@lo:38:kworker.0 00000100:00080000:1.0:1767320857.542592:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320857.542595:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320857.542597:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983175808:0@lo:38:kworker.0 00000100:00080000:3.0:1767320857.543131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320857.543143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320857.543147:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320857.543154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6100 00000100:00080000:3.0:1767320857.543158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320857.543164:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320857.543199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3480 x1853163983176192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320857.543482:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983175936 00000100:00100000:0.0:1767320857.543491:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983175936 00000100:00100000:0.0:1767320857.543541:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176064 00000100:00100000:0.0:1767320857.543544:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176064 00000100:00100000:0.0:1767320857.543587:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1205 00000100:00100000:0.0:1767320857.543595:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983175936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320857.543657:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983175936:12345-0@lo:400:kworker.0 Request processed in 63us (1987us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320857.543664:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1205 00000100:00100000:0.0:1767320857.543675:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1206 00000100:00100000:0.0:1767320857.543680:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983176064:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320857.543703:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983176064:12345-0@lo:400:kworker.0 Request processed in 23us (1688us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320857.543707:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1206 00000100:00100000:0.0:1767320857.543741:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983175936:0@lo:400:kworker.0 00000100:00100000:2.0:1767320857.543820:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983176064:0@lo:400:kworker.0 00000100:00100000:1.0:1767320857.544062:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983176192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320857.544133:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320857.544191:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176192 00000100:00100000:2.0:1767320857.544197:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176192 00000100:00100000:2.0:1767320857.544227:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2603 00000100:00100000:2.0:1767320857.544232:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176192:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320857.544247:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320857.544444:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176192:12345-0@lo:38:kworker.0 Request processed in 210us (311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320857.544452:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2603 00000100:00080000:0.0:1767320857.544525:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320857.544531:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320857.544536:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983176192:0@lo:38:kworker.0 00010000:00010000:3.0:1767320857.732639:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320858.756483:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320859.532977:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320859.533063:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320859.533142:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705088 00000100:00100000:2.0:1767320859.533161:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705088 00000100:00100000:2.0:1767320859.533202:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705216 00000100:00100000:2.0:1767320859.533205:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705216 00000100:00100000:3.0:1767320859.533224:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2604 00000100:00100000:3.0:1767320859.533234:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915705088:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:2.0:1767320859.533250:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1967 00010000:00020000:3.0:1767320859.533259:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320859.533262:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915705216:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320859.533348:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915705088:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 114us (384us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320859.533353:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2604 00000100:00080000:0.0:1767320859.533386:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320859.533654:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915705216:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 395us (590us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320859.533662:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1967 00010000:00010000:2.0:1767320859.785961:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320860.805232:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320861.829197:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320862.149237:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983176320:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320862.149331:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320862.149542:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176320 00000100:00100000:1.0:1767320862.149551:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176320 00000100:00100000:1.0:1767320862.149625:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1314 00000100:00100000:1.0:1767320862.149635:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983176320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320862.149664:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320862.149711:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983176320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320862.149718:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1314 00000100:00100000:1.0:1767320862.149765:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983176320:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320862.149876:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320862.149887:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983176448:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320862.149916:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320862.149989:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176448 00000100:00100000:3.0:1767320862.149993:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176448 00000100:00100000:3.0:1767320862.150028:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1315 00000100:00100000:3.0:1767320862.150034:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983176448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320862.150050:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320862.150082:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983176448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320862.150088:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1315 00000100:00100000:3.0:1767320862.150112:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983176448:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320862.150393:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320862.150398:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320862.660413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320862.660424:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320862.660553:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983176576:0@lo:400:kworker.0 00000100:00100000:2.0:1767320862.660620:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320862.660733:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176576 00000100:00100000:1.0:1767320862.660742:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176576 00000100:00100000:1.0:1767320862.660808:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968 00000100:00100000:1.0:1767320862.660827:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983176576:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320862.660938:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320862.660962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320862.660967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320862.660970:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320862.660977:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6106 00000100:00080000:3.0:1767320862.660982:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320862.660988:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320862.661011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3800 x1853163983176704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320862.661054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320862.661059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320862.661060:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320862.661064:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6106 00000100:00080000:3.0:1767320862.661066:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320862.661070:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320862.661081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163983176960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320862.661091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320862.661093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320862.661095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320862.661098:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6106 00000100:00080000:3.0:1767320862.661100:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320862.661104:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320862.661115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3480 x1853163983177088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320862.661307:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983176704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.661340:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.661366:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983176960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.661384:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.661398:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983177088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.661418:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.661449:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176704 00000100:00100000:0.0:1767320862.661453:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176704 00000100:00100000:0.0:1767320862.661487:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176960 00000100:00100000:0.0:1767320862.661491:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176960 00000100:00100000:0.0:1767320862.661507:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2605 00000100:00100000:0.0:1767320862.661512:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320862.661544:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320862.661578:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176704:12345-0@lo:38:kworker.0 Request processed in 64us (237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320862.661585:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2605 00000100:00100000:0.0:1767320862.661601:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2606 00000100:00100000:0.0:1767320862.661605:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176960:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320862.661614:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320862.661634:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983176960:12345-0@lo:38:kworker.0 Request processed in 28us (249us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320862.661640:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2606 00000100:00080000:0.0:1767320862.661668:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320862.661673:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320862.661677:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983176704:0@lo:38:kworker.0 00000100:00080000:0.0:1767320862.661685:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320862.661688:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320862.661690:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983176960:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.661727:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983177216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.661752:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.661773:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320862.661779:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983176832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.661796:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.661832:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177216 00000100:00100000:0.0:1767320862.661836:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177216 00000100:00100000:0.0:1767320862.661865:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1207 00000100:00100000:0.0:1767320862.661872:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983177216:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.661910:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983177216:12345-0@lo:400:kworker.0 Request processed in 38us (157us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320862.661916:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1207 00000100:00100000:0.0:1767320862.661931:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983176832 00000100:00100000:0.0:1767320862.661934:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983176832 00000100:00100000:0.0:1767320862.661954:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1969 00000100:00100000:0.0:1767320862.661959:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983176832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.661986:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983176832:12345-0@lo:400:kworker.0 Request processed in 26us (188us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320862.661991:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1969 00000100:00080000:0.0:1767320862.662004:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320862.662021:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3b80 x1853163983176832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320878 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320862.662032:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983176832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.662042:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983177216:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.662075:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177088 00000100:00100000:0.0:1767320862.662078:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177088 00000100:00100000:0.0:1767320862.662097:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 676 00000100:00100000:0.0:1767320862.662101:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983177088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320862.662113:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320862.662143:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983177088:12345-0@lo:38:kworker.0 Request processed in 41us (725us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320862.662149:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 676 00000100:00080000:0.0:1767320862.662170:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320862.662174:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320862.662177:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983177088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.662222:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983177344:0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.662245:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320862.662267:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177344 00000100:00100000:0.0:1767320862.662301:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177344 00000100:00100000:0.0:1767320862.662322:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1208 00000100:00100000:0.0:1767320862.662328:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983177344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320862.662358:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983177344:12345-0@lo:400:kworker.0 Request processed in 30us (112us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320862.662363:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1208 00000100:00100000:0.0:1767320862.662392:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983177344:0@lo:400:kworker.0 00000100:00100000:1.0:1767320862.669194:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983176576:12345-0@lo:400:kworker.0 Request processed in 8358us (8562us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320862.669211:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968 00000100:00080000:3.0:1767320862.670675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320862.670683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320862.670688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320862.670695:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6106 00000100:00080000:3.0:1767320862.670699:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320862.670706:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320862.670745:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c2d80 x1853163983177472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320862.670852:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983177472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320862.670917:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320862.671016:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177472 00000100:00100000:1.0:1767320862.671024:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177472 00000100:00100000:1.0:1767320862.671069:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2607 00000100:00100000:1.0:1767320862.671076:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983177472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320862.671096:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320862.671149:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983177472:12345-0@lo:38:kworker.0 Request processed in 72us (231us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320862.671156:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2607 00000100:00080000:3.0:1767320862.671253:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320862.671258:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320862.671262:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983177472:0@lo:38:kworker.0 00010000:00080000:2.0:1767320862.672679:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c1880 x1853163983176576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320878 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320862.672694:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983176576:0@lo:400:kworker.0 00010000:00010000:2.0:1767320862.855261:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320863.876604:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320864.653575:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:3.0:1767320864.653621:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915705344 offset 0 length 520: 4 00000100:00100000:3.0:1767320864.653648:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320864.653731:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705600 00000100:00100000:1.0:1767320864.653751:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705600 00000100:00100000:1.0:1767320864.653838:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705472 00000100:00100000:1.0:1767320864.653842:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705472 00000100:00100000:1.0:1767320864.653861:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1970 00000100:00100000:1.0:1767320864.653873:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915705600:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320864.653955:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915705600:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320864.653963:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1970 00000100:00100000:1.0:1767320864.653986:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2608 00000100:00100000:1.0:1767320864.653991:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915705472:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320864.654026:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320864.654061:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915705472:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 59us (403us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320864.654068:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2608 00000100:00080000:2.0:1767320864.658192:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320864.900605:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320865.926093:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320866.948700:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320867.268789:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983177600:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320867.268819:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320867.268836:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983177728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320867.268877:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320867.268911:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177600 00000100:00100000:2.0:1767320867.268919:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177600 00000100:00100000:2.0:1767320867.268973:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1316 00000100:00100000:2.0:1767320867.268981:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983177600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320867.269011:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767320867.269227:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320867.269249:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.269362:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177728 00000100:00100000:1.0:1767320867.269367:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177728 00000100:00100000:1.0:1767320867.269414:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767320867.269434:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983177728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320867.269454:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320867.269509:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983177728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (261us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320867.269516:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767320867.269560:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983177728:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320867.269627:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320867.273767:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983177600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4776us (4881us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320867.273782:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1316 00000100:00100000:3.0:1767320867.274604:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983177600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320867.780561:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320842/real 1767320842] req@ffff9346076f8e00 x1853163983172096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320867 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320867.780598:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076f8e00 x1853163983172096/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320867 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767320867.780619:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320867.780628:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320867.780632:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983172096:0@lo:250:kworker.0 00000100:00080000:0.0:1767320867.780696:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320867.780701:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320867.780704:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320867.780708:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6090 00000100:00080000:0.0:1767320867.780712:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320867.780730:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320867.780763:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5f80 x1853163983177856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320867.780843:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320867.780847:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320867.780849:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320867.780853:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6111 00000100:00080000:0.0:1767320867.780856:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320867.780860:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320867.780875:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163983178112/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320867.780910:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320867.780913:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320867.780915:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320867.780918:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6111 00000100:00080000:0.0:1767320867.780921:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320867.780924:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320867.780939:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015500 x1853163983178368/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320867.780946:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320867.780949:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320867.780950:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320867.780953:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6111 00000100:00080000:0.0:1767320867.780955:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320867.780958:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320867.780967:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017800 x1853163983178496/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320867.780988:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320867.780990:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320867.780992:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320867.780995:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6111 00000100:00080000:0.0:1767320867.780997:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320867.781000:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320867.781009:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163983178880/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320867.781033:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983178240:0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.781109:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320867.781170:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320867.781176:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983178624:0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.781202:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320867.781242:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178240 00000100:00100000:0.0:1767320867.781250:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178240 00000100:00100000:0.0:1767320867.781328:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971 00000100:00100000:0.0:1767320867.781337:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983178240:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.781399:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983178240:12345-0@lo:400:kworker.0 Request processed in 63us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320867.781407:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971 00000100:00100000:0.0:1767320867.781431:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178624 00000100:00100000:0.0:1767320867.781434:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178624 00000100:00100000:0.0:1767320867.781458:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1209 00000100:00100000:0.0:1767320867.781464:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983178624:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.781493:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983178624:12345-0@lo:400:kworker.0 Request processed in 29us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320867.781498:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1209 00000100:00080000:0.0:1767320867.781521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320867.781545:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983178624:0@lo:400:kworker.0 00010000:00080000:0.0:1767320867.781554:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5180 x1853163983178240/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320883 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320867.781564:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983178240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320867.781894:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983177856:0@lo:250:kworker.0 00000400:00000100:3.0:1767320867.781923:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983177856 offset 0 length 520: 4 00000100:00100000:3.0:1767320867.781945:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983178112:0@lo:38:kworker.0 00000100:00100000:3.0:1767320867.781965:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320867.781987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983178368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320867.782006:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320867.782018:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983178496:0@lo:38:kworker.0 00000100:00100000:3.0:1767320867.782048:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320867.782062:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983178880:0@lo:38:kworker.0 00000100:00100000:3.0:1767320867.782077:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.782240:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983177984:0@lo:400:kworker.0 00000100:00100000:1.0:1767320867.782290:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.782316:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320867.782322:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983178752:0@lo:400:kworker.0 00000100:00100000:1.0:1767320867.782344:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320867.782461:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178112 00000100:00100000:1.0:1767320867.782466:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178112 00000100:00100000:1.0:1767320867.782487:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178368 00000100:00100000:1.0:1767320867.782490:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178368 00000100:00100000:1.0:1767320867.782497:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178880 00000100:00100000:1.0:1767320867.782499:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178880 00000100:00100000:1.0:1767320867.782508:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983177984 00000100:00100000:1.0:1767320867.782510:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983177984 00000100:00100000:1.0:1767320867.782531:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1972 00000100:00100000:1.0:1767320867.782537:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983177984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320867.782575:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983177984:12345-0@lo:400:kworker.0 Request processed in 38us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320867.782581:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1972 00000100:00100000:1.0:1767320867.782596:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2609 00000100:00100000:1.0:1767320867.782599:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320867.782616:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320867.782649:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178112:12345-0@lo:38:kworker.0 Request processed in 47us (682us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.782655:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2609 00000100:00100000:1.0:1767320867.782667:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2610 00000100:00100000:1.0:1767320867.782671:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178368:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320867.782680:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320867.782704:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178368:12345-0@lo:38:kworker.0 Request processed in 31us (696us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.782709:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2610 00000100:00100000:1.0:1767320867.782786:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2611 00000100:00100000:1.0:1767320867.782791:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320867.782800:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320867.782913:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983178880:12345-0@lo:38:kworker.0 Request processed in 121us (834us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320867.782919:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2611 00000100:00080000:1.0:1767320867.782949:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320867.782954:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320867.782957:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983178112:0@lo:38:kworker.0 00000100:00080000:1.0:1767320867.782966:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320867.782969:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320867.782972:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983178368:0@lo:38:kworker.0 00000100:00080000:1.0:1767320867.782979:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320867.782982:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320867.782985:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983178880:0@lo:38:kworker.0 00010000:00080000:1.0:1767320867.783020:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee5880 x1853163983177984/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320883 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320867.783030:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983177984:0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.783161:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178496 00000100:00100000:0.0:1767320867.783167:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178496 00000100:00100000:0.0:1767320867.783193:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 677 00000100:00100000:0.0:1767320867.783198:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983178496:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320867.783208:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320867.783242:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983178496:12345-0@lo:38:kworker.0 Request processed in 43us (1193us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320867.783248:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 677 00000100:00100000:0.0:1767320867.783292:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983178752 00000100:00100000:0.0:1767320867.783296:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983178752 00000100:00100000:0.0:1767320867.783321:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1210 00000100:00100000:0.0:1767320867.783327:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983178752:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320867.783359:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983178752:12345-0@lo:400:kworker.0 Request processed in 32us (1013us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320867.783365:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1210 00000100:00080000:0.0:1767320867.783395:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320867.783435:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320867.783439:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320867.783442:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983178496:0@lo:38:kworker.0 00000100:00100000:1.0:1767320867.783653:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983178752:0@lo:400:kworker.0 00010000:00010000:3.0:1767320867.972606:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320868.996614:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320869.775526:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320869.775576:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320869.775684:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705856 00000100:00100000:0.0:1767320869.775695:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705856 00000100:00100000:0.0:1767320869.775772:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705728 00000100:00100000:0.0:1767320869.775775:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705728 00000100:00100000:0.0:1767320869.775798:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1973 00000100:00100000:0.0:1767320869.775809:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915705856:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320869.775882:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915705856:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 75us (367us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320869.775889:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1973 00000100:00100000:0.0:1767320869.775908:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2612 00000100:00100000:0.0:1767320869.775912:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915705728:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320869.775931:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320869.775957:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915705728:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (380us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320869.775964:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2612 00000100:00080000:2.0:1767320869.776168:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320870.021696:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320871.044619:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320872.070246:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320872.388479:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320872.388503:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983179008:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320872.388570:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983179136:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320872.388580:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.388610:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.388840:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179008 00000100:00100000:1.0:1767320872.388848:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179008 00000100:00100000:1.0:1767320872.388910:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179136 00000100:00100000:1.0:1767320872.388914:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179136 00000100:00100000:1.0:1767320872.388933:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1318 00000100:00100000:1.0:1767320872.388942:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983179008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320872.388973:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767320872.389521:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320872.389526:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320872.389554:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1319 00000100:00100000:3.0:1767320872.389563:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983179136:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320872.389581:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320872.398103:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983179008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 9150us (9516us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320872.398122:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1318 00000100:00100000:3.0:1767320872.398700:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983179136:12345-0@lo:13:osp-pre-0-1.0 Request processed in 8793us (9743us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320872.398711:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1319 00000100:00100000:0.0:1767320872.401510:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983179136:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320872.401723:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983179008:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320872.900560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320872.900573:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320872.900642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320872.900646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320872.900650:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320872.900657:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6116 00000100:00080000:3.0:1767320872.900662:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320872.900668:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320872.900689:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346392c3100 x1853163983179392/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320872.900735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320872.900739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320872.900741:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320872.900744:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6116 00000100:00080000:3.0:1767320872.900747:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320872.900751:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320872.900764:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc700 x1853163983179648/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320872.900772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320872.900775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320872.900777:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320872.900780:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6116 00000100:00080000:3.0:1767320872.900782:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320872.900786:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320872.900795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddc00 x1853163983179776/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320872.900815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320872.900818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320872.900820:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320872.900822:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6116 00000100:00080000:3.0:1767320872.900825:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320872.900828:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320872.900837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163983180160/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320872.900864:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983179904:0@lo:400:kworker.0 00000100:00100000:3.0:1767320872.900926:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320872.900959:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320872.900965:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983179520:0@lo:400:kworker.0 00000100:00100000:3.0:1767320872.900990:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.901192:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983179392:0@lo:38:kworker.0 00000100:00100000:1.0:1767320872.901230:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.901251:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983179648:0@lo:38:kworker.0 00000100:00100000:1.0:1767320872.901267:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.901337:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983179776:0@lo:38:kworker.0 00000100:00100000:1.0:1767320872.901358:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.901371:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983180160:0@lo:38:kworker.0 00000100:00100000:1.0:1767320872.901387:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320872.901414:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179520 00000100:00100000:1.0:1767320872.901421:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179520 00000100:00100000:1.0:1767320872.901468:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179392 00000100:00100000:1.0:1767320872.901471:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179392 00000100:00100000:1.0:1767320872.901479:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179648 00000100:00100000:1.0:1767320872.901481:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179648 00000100:00100000:1.0:1767320872.901486:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180160 00000100:00100000:1.0:1767320872.901488:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180160 00000100:00100000:1.0:1767320872.901498:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1974 00000100:00100000:1.0:1767320872.901506:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983179520:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320872.901551:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983179520:12345-0@lo:400:kworker.0 Request processed in 46us (560us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320872.901558:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1974 00000100:00100000:1.0:1767320872.901574:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2613 00000100:00100000:1.0:1767320872.901578:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983179392:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320872.901594:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320872.901637:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983179392:12345-0@lo:38:kworker.0 Request processed in 57us (406us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320872.901643:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2613 00000100:00100000:1.0:1767320872.901653:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2614 00000100:00100000:1.0:1767320872.901657:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983179648:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320872.901665:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320872.901684:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983179648:12345-0@lo:38:kworker.0 Request processed in 26us (416us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320872.901689:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2614 00000100:00100000:1.0:1767320872.901698:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2615 00000100:00100000:1.0:1767320872.901702:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983180160:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320872.901709:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320872.901729:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983180160:12345-0@lo:38:kworker.0 Request processed in 26us (341us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320872.901734:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2615 00000100:00080000:1.0:1767320872.901749:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320872.901775:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320872.901779:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.901782:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983179392:0@lo:38:kworker.0 00000100:00080000:1.0:1767320872.901790:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320872.901792:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.901794:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983179648:0@lo:38:kworker.0 00000100:00080000:1.0:1767320872.901800:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320872.901803:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320872.901805:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983180160:0@lo:38:kworker.0 00010000:00080000:1.0:1767320872.901837:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3800 x1853163983179520/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320888 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320872.901847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983179520:0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902145:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179904 00000100:00100000:0.0:1767320872.902150:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179904 00000100:00100000:0.0:1767320872.902191:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1211 00000100:00100000:0.0:1767320872.902199:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983179904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902252:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983179904:12345-0@lo:400:kworker.0 Request processed in 55us (1326us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320872.902260:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1211 00000100:00100000:0.0:1767320872.902310:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179776 00000100:00100000:0.0:1767320872.902314:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179776 00000100:00100000:0.0:1767320872.902333:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 678 00000100:00100000:0.0:1767320872.902338:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983179776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320872.902348:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320872.902379:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983179776:12345-0@lo:38:kworker.0 Request processed in 40us (1019us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320872.902384:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 678 00000100:00100000:0.0:1767320872.902416:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983179904:0@lo:400:kworker.0 00000100:00080000:0.0:1767320872.902436:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320872.902439:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320872.902442:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983179776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320872.902457:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983180032:0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902485:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.902504:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180032 00000100:00100000:0.0:1767320872.902507:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180032 00000100:00100000:0.0:1767320872.902528:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1212 00000100:00100000:0.0:1767320872.902533:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983180032:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902559:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983180032:12345-0@lo:400:kworker.0 Request processed in 26us (73us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320872.902565:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1212 00000100:00100000:0.0:1767320872.902592:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983180032:0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902610:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320872.902617:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983179264:0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902638:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320872.902657:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983179264 00000100:00100000:0.0:1767320872.902660:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983179264 00000100:00100000:0.0:1767320872.902688:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1975 00000100:00100000:0.0:1767320872.902694:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983179264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320872.902731:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983179264:12345-0@lo:400:kworker.0 Request processed in 36us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320872.902737:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1975 00000100:00080000:0.0:1767320872.902753:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320872.902768:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346392c3b80 x1853163983179264/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320888 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320872.902779:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983179264:0@lo:400:kworker.0 00010000:00010000:3.0:1767320873.092810:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320874.116474:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320874.897105:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320874.897172:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320874.902028:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706112 00000100:00100000:1.0:1767320874.902045:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706112 00000100:00100000:1.0:1767320874.902134:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915705984 00000100:00100000:1.0:1767320874.902138:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915705984 00000100:00100000:1.0:1767320874.902169:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1976 00000100:00100000:1.0:1767320874.902181:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915706112:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320874.902296:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915706112:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 116us (5198us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320874.902304:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1976 00000100:00100000:1.0:1767320874.902327:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2616 00000100:00100000:1.0:1767320874.902333:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915705984:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320874.902352:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320874.902379:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915705984:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (5207us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320874.902385:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2616 00000100:00080000:0.0:1767320874.903336:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320875.140735:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320876.164873:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320877.188445:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320877.509770:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320877.509795:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983180288:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320877.509863:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320877.511726:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180288 00000100:00100000:2.0:1767320877.511741:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180288 00000100:00100000:2.0:1767320877.511833:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1320 00000100:00100000:2.0:1767320877.511843:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983180288:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320877.511888:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320877.511965:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983180288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (2100us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320877.511974:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1320 00000100:00080000:3.0:1767320877.512243:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320877.512315:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983180288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320877.516860:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320877.522778:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983180416:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320877.522866:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320877.523980:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180416 00000100:00100000:2.0:1767320877.523990:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180416 00000100:00100000:2.0:1767320877.524107:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1321 00000100:00100000:2.0:1767320877.524117:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983180416:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320877.524150:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320877.524303:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983180416:12345-0@lo:13:osp-pre-0-1.0 Request processed in 187us (1436us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320877.524311:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1321 00000100:00100000:1.0:1767320877.525618:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983180416:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320877.527056:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320878.020414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320878.020426:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320878.020490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320878.020495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320878.020499:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320878.020507:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6121 00000100:00080000:3.0:1767320878.020512:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320878.020519:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320878.020542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382de680 x1853163983180672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320878.020570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983180544:0@lo:400:kworker.0 00000100:00080000:3.0:1767320878.020597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320878.020601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320878.020603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320878.020608:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6121 00000100:00080000:3.0:1767320878.020610:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320878.020615:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320878.020628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc700 x1853163983180928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320878.020639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1767320878.020642:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320878.020644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320878.020648:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320878.020652:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6121 00000100:00080000:3.0:1767320878.020655:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320878.020659:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320878.020674:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594a80 x1853163983181056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320878.020684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320878.020687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320878.020689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320878.020693:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6121 00000100:00080000:3.0:1767320878.020696:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320878.020699:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320878.020710:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597480 x1853163983181184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320878.020726:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983180800:0@lo:400:kworker.0 00000100:00100000:1.0:1767320878.020756:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180544 00000100:00100000:2.0:1767320878.020761:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320878.020766:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180544 00000100:00100000:1.0:1767320878.020819:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180800 00000100:00100000:1.0:1767320878.020823:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180800 00000100:00100000:1.0:1767320878.020846:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1977 00000100:00100000:1.0:1767320878.020854:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983180544:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320878.020905:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983180544:12345-0@lo:400:kworker.0 Request processed in 51us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320878.020913:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1977 00000100:00100000:2.0:1767320878.020926:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1978 00000100:00100000:2.0:1767320878.020938:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983180800:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320878.020950:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320878.020969:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983180672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320878.020982:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983180800:12345-0@lo:400:kworker.0 Request processed in 47us (222us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320878.020991:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1978 00010000:00080000:3.0:1767320878.021009:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ded80 x1853163983180800/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320894 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320878.021024:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983180800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320878.025151:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320878.025224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983180928:0@lo:38:kworker.0 00000100:00100000:3.0:1767320878.025256:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180672 00000100:00100000:3.0:1767320878.025267:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180672 00000100:00100000:0.0:1767320878.025288:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320878.025308:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983181056:0@lo:38:kworker.0 00000100:00100000:0.0:1767320878.025342:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320878.025381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983181184:0@lo:38:kworker.0 00000100:00100000:0.0:1767320878.025401:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320878.025438:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181056 00000100:00100000:0.0:1767320878.025443:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181056 00000100:00100000:0.0:1767320878.025473:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 679 00000100:00100000:0.0:1767320878.025480:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983181056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320878.025501:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320878.025543:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983181056:12345-0@lo:38:kworker.0 Request processed in 62us (200us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320878.025550:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 679 00000100:00100000:0.0:1767320878.025573:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983180928 00000100:00100000:0.0:1767320878.025577:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983180928 00000100:00100000:0.0:1767320878.025591:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181184 00000100:00100000:0.0:1767320878.025593:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181184 00000100:00100000:0.0:1767320878.025615:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2617 00000100:00100000:0.0:1767320878.025620:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983180928:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320878.025631:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320878.025658:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983180928:12345-0@lo:38:kworker.0 Request processed in 36us (388us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320878.025660:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2618 00000100:00100000:0.0:1767320878.025670:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2617 00000100:00100000:2.0:1767320878.025673:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983181184:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320878.025687:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320878.025735:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2619 00000100:00100000:3.0:1767320878.025741:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983180672:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320878.025751:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320878.025930:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320878.025936:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320878.025940:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983180928:0@lo:38:kworker.0 00010000:00080000:1.0:1767320878.028060:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382ddc00 x1853163983180544/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320894 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320878.028082:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983180544:0@lo:400:kworker.0 00000100:00080000:0.0:1767320878.028628:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320878.028635:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320878.028639:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983181056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320878.028788:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983181184:12345-0@lo:38:kworker.0 Request processed in 3114us (3382us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320878.028799:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2618 00000100:00080000:0.0:1767320878.028817:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320878.028823:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320878.028828:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983181184:0@lo:38:kworker.0 00000100:00100000:3.0:1767320878.028895:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983180672:12345-0@lo:38:kworker.0 Request processed in 3152us (3750us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320878.028905:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2619 00000100:00080000:0.0:1767320878.028924:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320878.028929:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320878.028934:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983180672:0@lo:38:kworker.0 00010000:00010000:2.0:1767320878.215327:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320879.239410:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320880.012759:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320880.012809:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320880.012908:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706368 00000100:00100000:2.0:1767320880.012919:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706368 00000100:00100000:2.0:1767320880.012997:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706240 00000100:00100000:2.0:1767320880.013000:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706240 00000100:00100000:2.0:1767320880.013017:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1979 00000100:00100000:2.0:1767320880.013027:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915706368:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320880.013113:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915706368:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (364us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320880.013120:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1979 00000100:00100000:2.0:1767320880.013136:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2620 00000100:00100000:2.0:1767320880.013140:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915706240:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320880.013170:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320880.013194:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915706240:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (385us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320880.013200:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2620 00000100:00080000:0.0:1767320880.013459:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320880.260971:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320881.284610:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320882.308512:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320882.628574:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320882.628597:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983181312:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320882.628664:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320882.628689:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181312 00000100:00100000:0.0:1767320882.628699:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181312 00000100:00080000:3.0:1767320882.628742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320882.628770:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1322 00000100:00100000:0.0:1767320882.628779:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983181312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320882.628808:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320882.628862:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983181312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (198us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320882.628870:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1322 00000100:00100000:1.0:1767320882.629101:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320882.629110:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983181312:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320882.629111:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983181440:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320882.629139:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320882.629214:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181440 00000100:00100000:2.0:1767320882.629220:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181440 00000100:00100000:2.0:1767320882.629259:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1323 00000100:00100000:2.0:1767320882.629266:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983181440:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320882.629288:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767320882.629331:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320882.629379:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983181440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 113us (239us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320882.629386:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1323 00000100:00100000:0.0:1767320882.629399:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983181440:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320883.140649:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320883.140661:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320883.140744:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320883.140749:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320883.140753:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320883.140761:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6126 00000100:00080000:0.0:1767320883.140766:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320883.140773:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320883.140797:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163983181696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320883.140838:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1767320883.140845:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320883.140850:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320883.140852:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320883.140856:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983181568:0@lo:400:kworker.0 00000100:00080000:0.0:1767320883.140857:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6126 00000100:00080000:0.0:1767320883.140862:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320883.140868:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320883.140884:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5880 x1853163983181952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320883.140894:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320883.140898:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320883.140899:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320883.140903:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6126 00000100:00080000:0.0:1767320883.140906:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320883.140912:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320883.140925:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163983182080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320883.140946:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320883.140949:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320883.140951:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320883.140954:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6126 00000100:00080000:0.0:1767320883.140956:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320883.140960:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320883.140969:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5180 x1853163983182208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320883.140989:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320883.140997:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983181824:0@lo:400:kworker.0 00000100:00100000:3.0:1767320883.141231:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983181696:0@lo:38:kworker.0 00000100:00100000:2.0:1767320883.141454:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320883.141939:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181568 00000100:00100000:3.0:1767320883.141948:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181568 00000100:00100000:3.0:1767320883.142063:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1980 00000100:00100000:1.0:1767320883.142064:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320883.142075:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983181568:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320883.142119:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181824 00000100:00100000:1.0:1767320883.142124:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181824 00000100:00100000:3.0:1767320883.142175:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983181568:12345-0@lo:400:kworker.0 Request processed in 102us (720us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320883.142186:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320883.142187:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1980 00000100:00100000:3.0:1767320883.142208:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181696 00000100:00100000:3.0:1767320883.142212:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181696 00000100:00100000:0.0:1767320883.142231:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983181952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320883.142257:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2621 00000100:00100000:3.0:1767320883.142266:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983181696:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320883.142267:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320883.142312:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983182080:0@lo:38:kworker.0 00010000:00020000:3.0:1767320883.142319:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320883.142352:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320883.142365:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983181696:12345-0@lo:38:kworker.0 Request processed in 99us (181us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320883.142374:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2621 00000100:00100000:0.0:1767320883.142382:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983182208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320883.142387:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983181952 00000100:00100000:3.0:1767320883.142392:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983181952 00000100:00100000:3.0:1767320883.142410:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2622 00000100:00100000:0.0:1767320883.142412:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320883.142419:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983181952:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320883.142440:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320883.142462:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767320883.142468:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983181952:12345-0@lo:38:kworker.0 Request processed in 49us (203us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767320883.142471:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320883.142479:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2622 00000100:00100000:0.0:1767320883.142481:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983181696:0@lo:38:kworker.0 00000100:00100000:3.0:1767320883.142490:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182208 00000100:00100000:3.0:1767320883.142495:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182208 00000100:00080000:0.0:1767320883.142497:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320883.142502:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320883.142505:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983181952:0@lo:38:kworker.0 00000100:00100000:3.0:1767320883.142523:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2623 00000100:00100000:3.0:1767320883.142528:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320883.142537:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320883.142541:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182080 00000100:00100000:0.0:1767320883.142546:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182080 00000100:00100000:3.0:1767320883.142570:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eaa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182208:12345-0@lo:38:kworker.0 Request processed in 40us (161us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320883.142575:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 680 00000100:00100000:0.0:1767320883.142585:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983182080:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320883.142586:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2623 00010000:00020000:0.0:1767320883.142600:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00080000:3.0:1767320883.142889:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016300 x1853163983181568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320899 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320883.142901:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983181568:0@lo:400:kworker.0 00000100:00080000:2.0:1767320883.143355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320883.143752:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983182080:12345-0@lo:38:kworker.0 Request processed in 1167us (1399us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320883.143762:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 680 00000100:00080000:3.0:1767320883.143836:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320883.143841:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320883.143845:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983182208:0@lo:38:kworker.0 00000100:00080000:3.0:1767320883.143865:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320883.143869:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320883.143872:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983182080:0@lo:38:kworker.0 00000100:00100000:1.0:1767320883.145068:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1981 00000100:00100000:1.0:1767320883.145080:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983181824:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320883.145184:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983181824:12345-0@lo:400:kworker.0 Request processed in 106us (3130us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320883.145192:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1981 00010000:00080000:0.0:1767320883.145361:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015500 x1853163983181824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320899 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320883.145376:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983181824:0@lo:400:kworker.0 00010000:00010000:3.0:1767320883.332735:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320884.360601:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320885.134209:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320885.134331:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320885.134640:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706496 00000100:00100000:3.0:1767320885.134651:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706496 00000100:00100000:3.0:1767320885.134703:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706624 00000100:00100000:3.0:1767320885.134706:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706624 00000100:00100000:3.0:1767320885.134760:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1982 00000100:00100000:3.0:1767320885.134772:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915706624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320885.134843:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915706624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 74us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320885.134852:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1982 00000100:00100000:3.0:1767320885.134870:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2624 00000100:00100000:3.0:1767320885.134875:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915706496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320885.134896:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320885.134920:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915706496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (722us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320885.134928:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2624 00000100:00080000:1.0:1767320885.138315:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320885.386367:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320886.405522:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320887.428557:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320887.748853:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320887.748880:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983182336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320887.748952:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320887.749304:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182336 00000100:00100000:2.0:1767320887.749314:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182336 00000100:00100000:2.0:1767320887.749387:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1324 00000100:00100000:2.0:1767320887.749396:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983182336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320887.749429:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320887.749481:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983182336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (527us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320887.749490:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320887.749490:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1324 00000100:00100000:1.0:1767320887.749501:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983182464:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320887.749532:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320887.749574:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182464 00000100:00100000:0.0:1767320887.749579:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182464 00000100:00100000:0.0:1767320887.749612:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1325 00000100:00100000:0.0:1767320887.749620:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983182464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320887.749653:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320887.749699:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983182464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320887.749705:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1325 00000100:00080000:3.0:1767320887.749844:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320887.749849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320887.749889:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983182336:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320887.750056:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983182464:0@lo:13:osp-pre-0-1.0 00000100:00080000:2.0:1767320888.260597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320888.260609:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767320888.260708:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983182592:0@lo:400:kworker.0 00000100:00100000:1.0:1767320888.260778:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320888.260921:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182592 00000100:00100000:3.0:1767320888.260929:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182592 00000100:00100000:3.0:1767320888.261017:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1983 00000100:00100000:3.0:1767320888.261026:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983182592:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320888.261099:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320888.261146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320888.261150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320888.261155:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320888.261159:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6131 00000100:00080000:2.0:1767320888.261163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320888.261168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320888.261192:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396f480 x1853163983182720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320888.261225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320888.261229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320888.261231:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320888.261234:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6131 00000100:00080000:2.0:1767320888.261237:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320888.261240:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320888.261250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163983182976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320888.261259:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320888.261262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320888.261263:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320888.261267:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6131 00000100:00080000:2.0:1767320888.261296:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320888.261301:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320888.261315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163983183104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320888.261324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320888.261327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320888.261329:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320888.261332:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6131 00000100:00080000:2.0:1767320888.261334:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320888.261337:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320888.261348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582d80 x1853163983183232/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320888.261371:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983182848:0@lo:400:kworker.0 00000100:00100000:2.0:1767320888.261402:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320888.261433:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983182720:0@lo:38:kworker.0 00000100:00100000:2.0:1767320888.261457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320888.261496:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983182976:0@lo:38:kworker.0 00000100:00100000:2.0:1767320888.261517:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320888.261535:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983183104:0@lo:38:kworker.0 00000100:00100000:2.0:1767320888.261558:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320888.261577:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983183232:0@lo:38:kworker.0 00000100:00100000:2.0:1767320888.261594:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320888.261619:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182848 00000100:00100000:2.0:1767320888.261624:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182848 00000100:00100000:2.0:1767320888.261648:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182720 00000100:00100000:2.0:1767320888.261651:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182720 00000100:00100000:2.0:1767320888.261659:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983182976 00000100:00100000:2.0:1767320888.261661:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983182976 00000100:00100000:2.0:1767320888.261668:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183232 00000100:00100000:2.0:1767320888.261670:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183232 00000100:00100000:2.0:1767320888.261678:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1984 00000100:00100000:2.0:1767320888.261684:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983182848:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320888.261721:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983182848:12345-0@lo:400:kworker.0 Request processed in 35us (318us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320888.261728:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1984 00000100:00100000:2.0:1767320888.261745:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2625 00000100:00100000:2.0:1767320888.261749:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320888.261767:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320888.261800:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182720:12345-0@lo:38:kworker.0 Request processed in 48us (341us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320888.261806:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2625 00000100:00100000:2.0:1767320888.261817:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2626 00000100:00100000:2.0:1767320888.261821:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182976:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320888.261829:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320888.261851:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983182976:12345-0@lo:38:kworker.0 Request processed in 29us (333us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320888.261857:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2626 00000100:00100000:2.0:1767320888.261867:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2627 00000100:00100000:2.0:1767320888.261871:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983183232:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320888.261880:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320888.261902:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983183232:12345-0@lo:38:kworker.0 Request processed in 30us (307us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320888.261908:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2627 00010000:00080000:2.0:1767320888.261943:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396c000 x1853163983182848/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320904 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320888.261956:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983182848:0@lo:400:kworker.0 00000100:00080000:2.0:1767320888.261986:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320888.262004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320888.262007:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983182720:0@lo:38:kworker.0 00000100:00080000:2.0:1767320888.262018:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320888.262021:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320888.262024:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983182976:0@lo:38:kworker.0 00000100:00080000:2.0:1767320888.262031:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320888.262034:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320888.262037:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983183232:0@lo:38:kworker.0 00000100:00100000:0.0:1767320888.262132:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183104 00000100:00100000:0.0:1767320888.262138:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183104 00000100:00100000:0.0:1767320888.262168:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 681 00000100:00100000:0.0:1767320888.262173:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983183104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320888.262183:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320888.262220:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983183104:12345-0@lo:38:kworker.0 Request processed in 46us (661us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320888.262226:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 681 00000100:00080000:0.0:1767320888.262253:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320888.262291:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320888.262295:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320888.262299:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983183104:0@lo:38:kworker.0 00000100:00100000:3.0:1767320888.262703:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983182592:12345-0@lo:400:kworker.0 Request processed in 1677us (1922us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320888.262711:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1983 00010000:00080000:1.0:1767320888.262795:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396f100 x1853163983182592/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320904 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320888.262806:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983182592:0@lo:400:kworker.0 00010000:00010000:3.0:1767320888.452840:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320889.228035:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320889.228520:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707008 00000100:00100000:3.0:1767320889.228530:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707008 00000100:00100000:3.0:1767320889.228605:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1985 00000100:00100000:3.0:1767320889.228616:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915707008:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767320889.228742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320889.228842:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915707008:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 228us (817us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320889.228848:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1985 00000400:00000100:1.0:1767320889.229032:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915706752 offset 0 length 520: 4 00000100:00100000:1.0:1767320889.229111:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320889.229352:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915706880 00000100:00100000:3.0:1767320889.229367:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915706880 00000100:00100000:3.0:1767320889.229449:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2628 00000100:00100000:3.0:1767320889.229455:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915706880:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320889.229475:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320889.229563:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915706880:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 106us (450us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320889.229570:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2628 00010000:00010000:3.0:1767320889.477295:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320890.500715:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320891.524675:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320892.356393:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320867/real 1767320867] req@ffff93462dee5f80 x1853163983177856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320892 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320892.356429:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93462dee5f80 x1853163983177856/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320892 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1767320892.356449:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320892.356480:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:0.0:1767320892.356484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983177856:0@lo:250:kworker.0 00000100:00080000:2.0:1767320892.357216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320892.357224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320892.357227:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320892.357232:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6116 00000100:00080000:2.0:1767320892.357237:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320892.357243:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320892.357297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a582300 x1853163983183360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320892.357364:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983183360:0@lo:250:kworker.0 00000400:00000100:0.0:1767320892.357410:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983183360 offset 0 length 520: 4 00000100:00080000:2.0:1767320892.357511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320892.357515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320892.357518:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320892.357522:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6136 00000100:00080000:2.0:1767320892.357524:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320892.357529:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320892.357545:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581880 x1853163983183616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320892.357592:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983183488:0@lo:400:kworker.0 00000100:00100000:1.0:1767320892.357633:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.361312:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183488 00000100:00100000:3.0:1767320892.361327:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183488 00000100:00100000:3.0:1767320892.361399:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1986 00000100:00100000:3.0:1767320892.361411:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983183488:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320892.361502:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983183488:12345-0@lo:400:kworker.0 Request processed in 94us (3868us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320892.361508:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1986 00000100:00080000:1.0:1767320892.361660:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320892.361706:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583800 x1853163983183488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320908 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320892.361730:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983183488:0@lo:400:kworker.0 00000100:00100000:0.0:1767320892.362350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983183616:0@lo:38:kworker.0 00000100:00100000:0.0:1767320892.362398:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.362500:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183616 00000100:00100000:3.0:1767320892.362506:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183616 00000100:00100000:3.0:1767320892.362542:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2629 00000100:00100000:3.0:1767320892.362548:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983183616:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320892.362569:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320892.362610:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983183616:12345-0@lo:38:kworker.0 Request processed in 60us (211us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320892.362616:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2629 00000100:00080000:1.0:1767320892.362694:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320892.362700:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320892.362705:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983183616:0@lo:38:kworker.0 00000100:00100000:1.0:1767320892.363338:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767320892.363357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320892.363362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320892.363364:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320892.363368:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6136 00000100:00080000:2.0:1767320892.363372:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320892.363377:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320892.363396:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581880 x1853163983183872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320892.363419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320892.363422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320892.363424:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320892.363428:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6136 00000100:00080000:2.0:1767320892.363430:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320892.363433:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320892.363444:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a583800 x1853163983184000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320892.363495:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983183872:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.363532:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.363552:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983184000:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.363576:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.363620:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184000 00000100:00100000:3.0:1767320892.363626:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184000 00000100:00100000:3.0:1767320892.363650:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 682 00000100:00100000:3.0:1767320892.363656:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983184000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320892.363670:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320892.363701:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983184000:12345-0@lo:38:kworker.0 Request processed in 44us (123us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320892.363707:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 682 00000100:00080000:3.0:1767320892.363731:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320892.363735:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320892.363738:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983184000:0@lo:38:kworker.0 00000100:00100000:0.0:1767320892.366324:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320892.366353:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983184128:0@lo:400:kworker.0 00000100:00100000:0.0:1767320892.366425:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320892.366493:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183872 00000100:00100000:0.0:1767320892.366501:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183872 00000100:00100000:0.0:1767320892.366535:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2630 00000100:00100000:0.0:1767320892.366543:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983183872:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320892.366565:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320892.366612:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983183872:12345-0@lo:38:kworker.0 Request processed in 68us (3078us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320892.366621:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2630 00000100:00100000:3.0:1767320892.366721:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184128 00000100:00100000:3.0:1767320892.366726:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184128 00000100:00080000:0.0:1767320892.366786:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320892.366800:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1213 00000100:00100000:3.0:1767320892.366810:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983184128:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767320892.366839:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320892.366846:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320892.366851:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983183872:0@lo:38:kworker.0 00000100:00100000:3.0:1767320892.366865:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983184128:12345-0@lo:400:kworker.0 Request processed in 57us (439us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320892.366873:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1213 00000100:00100000:3.0:1767320892.366910:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983184128:0@lo:400:kworker.0 00000100:00100000:1.0:1767320892.370310:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983183744:0@lo:400:kworker.0 00000100:00100000:1.0:1767320892.370383:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320892.370473:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983183744 00000100:00100000:0.0:1767320892.370497:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983183744 00000100:00080000:2.0:1767320892.370611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320892.370616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320892.370620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320892.370625:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6136 00000100:00080000:2.0:1767320892.370629:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320892.370634:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320892.370655:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ed80 x1853163983184384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320892.370692:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983184384:0@lo:38:kworker.0 00000100:00100000:2.0:1767320892.370721:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320892.370754:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184384 00000100:00100000:2.0:1767320892.370758:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184384 00000100:00100000:2.0:1767320892.370806:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2631 00000100:00100000:2.0:1767320892.370813:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983184384:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320892.370831:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320892.370850:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1987 00000100:00100000:2.0:1767320892.370871:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983184384:12345-0@lo:38:kworker.0 Request processed in 57us (149us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320892.370874:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983183744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320892.370883:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2631 00000100:00080000:2.0:1767320892.370936:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320892.370937:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a36680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983183744:12345-0@lo:400:kworker.0 Request processed in 67us (553us total) trans 0 rc 0/0 00000100:00080000:2.0:1767320892.370942:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320892.370950:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983184384:0@lo:38:kworker.0 00000100:00100000:0.0:1767320892.370955:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1987 00000100:00080000:0.0:1767320892.370975:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320892.371188:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320892.371198:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983184256:0@lo:400:kworker.0 00000100:00100000:1.0:1767320892.371230:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767320892.371292:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a583480 x1853163983183744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320908 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320892.371303:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983183744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320892.371984:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184256 00000100:00100000:0.0:1767320892.371991:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184256 00000100:00080000:1.0:1767320892.372067:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320892.386258:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1214 00000100:00100000:0.0:1767320892.386299:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983184256:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320892.386408:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983184256:12345-0@lo:400:kworker.0 Request processed in 138us (15176us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320892.386416:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1214 00000100:00100000:1.0:1767320892.389543:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983184256:0@lo:400:kworker.0 00010000:00010000:3.0:1767320892.548666:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320892.874090:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983184512:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320892.874177:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320892.874413:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184512 00000100:00100000:3.0:1767320892.874424:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184512 00000100:00100000:3.0:1767320892.874494:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1326 00000100:00100000:3.0:1767320892.874504:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983184512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767320892.874534:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320892.876700:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320892.876719:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983184640:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320892.876781:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320892.876838:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184640 00000100:00100000:2.0:1767320892.876847:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184640 00000100:00100000:2.0:1767320892.876888:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1327 00000100:00100000:2.0:1767320892.876896:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983184640:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320892.876932:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320892.876975:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983184640:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (193us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320892.876983:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1327 00000100:00100000:2.0:1767320892.877031:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983184640:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320892.878147:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983184512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3639us (3962us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320892.878158:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1326 00000100:00100000:0.0:1767320892.883156:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983184512:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320893.572433:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320894.349063:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320894.349164:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320894.349335:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707264 00000100:00100000:0.0:1767320894.349345:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707264 00000100:00100000:0.0:1767320894.349424:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707136 00000100:00100000:0.0:1767320894.349427:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707136 00000100:00100000:0.0:1767320894.349447:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1988 00000100:00100000:0.0:1767320894.349458:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915707264:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320894.349547:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915707264:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (504us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320894.349555:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1988 00000100:00100000:0.0:1767320894.349574:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2632 00000100:00100000:0.0:1767320894.349578:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915707136:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320894.349599:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320894.349629:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915707136:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 48us (465us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320894.349648:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2632 00000100:00080000:1.0:1767320894.350220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320894.601483:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320895.620793:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320896.644484:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320897.476984:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320897.476995:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320897.477089:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983184768:0@lo:400:kworker.0 00000100:00100000:2.0:1767320897.477157:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320897.477313:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320897.477319:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320897.477322:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320897.477327:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6140 00000100:00080000:3.0:1767320897.477331:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320897.477336:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320897.477359:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596300 x1853163983184896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320897.477391:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320897.477394:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320897.477395:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320897.477398:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6140 00000100:00080000:3.0:1767320897.477400:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320897.477402:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320897.477410:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595f80 x1853163983185152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320897.477416:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320897.477418:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320897.477420:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320897.477422:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6140 00000100:00080000:3.0:1767320897.477423:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320897.477426:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320897.477434:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597100 x1853163983185280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320897.477538:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184768 00000100:00100000:0.0:1767320897.477544:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184768 00000100:00100000:0.0:1767320897.477598:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1989 00000100:00100000:0.0:1767320897.477607:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983184768:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.477697:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983185536:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.477731:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.477764:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185536 00000100:00100000:1.0:1767320897.477768:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185536 00000100:00100000:1.0:1767320897.477809:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1215 00000100:00100000:1.0:1767320897.477816:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983185536:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.477894:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983185536:12345-0@lo:400:kworker.0 Request processed in 78us (161us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320897.477901:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1215 00000100:00080000:1.0:1767320897.477921:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320897.477925:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767320897.477950:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983185536:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.477987:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983185024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478010:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.478027:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320897.478032:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983185408:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478052:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.478079:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185024 00000100:00100000:1.0:1767320897.478083:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185024 00000100:00100000:1.0:1767320897.478112:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1990 00000100:00100000:1.0:1767320897.478118:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983185024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478149:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983185024:12345-0@lo:400:kworker.0 Request processed in 31us (137us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320897.478155:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1990 00000100:00080000:1.0:1767320897.478168:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320897.478185:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594700 x1853163983185024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320913 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320897.478194:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983185024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478208:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185408 00000100:00100000:1.0:1767320897.478211:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185408 00000100:00100000:1.0:1767320897.478249:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1216 00000100:00100000:1.0:1767320897.478255:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983185408:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478319:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983185408:12345-0@lo:400:kworker.0 Request processed in 65us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320897.478326:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1216 00000100:00080000:1.0:1767320897.478349:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320897.478364:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983185408:0@lo:400:kworker.0 00000100:00100000:1.0:1767320897.478422:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983184896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.478448:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.478465:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983185152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.478483:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.478498:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983185280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.478518:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320897.478542:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983184896 00000100:00100000:1.0:1767320897.478546:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983184896 00000100:00100000:1.0:1767320897.478561:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185152 00000100:00100000:1.0:1767320897.478563:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185152 00000100:00100000:1.0:1767320897.478572:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2633 00000100:00100000:1.0:1767320897.478576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983184896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320897.478591:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320897.478622:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983184896:12345-0@lo:38:kworker.0 Request processed in 44us (172us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320897.478628:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2633 00000100:00100000:1.0:1767320897.478639:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2634 00000100:00100000:1.0:1767320897.478643:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983185152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320897.478651:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320897.478674:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983185152:12345-0@lo:38:kworker.0 Request processed in 30us (190us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320897.478680:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2634 00000100:00080000:1.0:1767320897.478702:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320897.478707:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320897.478710:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983184896:0@lo:38:kworker.0 00000100:00080000:1.0:1767320897.478717:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320897.478720:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320897.478722:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983185152:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.478750:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185280 00000100:00100000:1.0:1767320897.478754:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185280 00000100:00100000:1.0:1767320897.478774:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 683 00000100:00100000:1.0:1767320897.478778:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983185280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320897.478788:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320897.478810:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983185280:12345-0@lo:38:kworker.0 Request processed in 31us (291us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320897.478814:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 683 00000100:00080000:1.0:1767320897.478846:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320897.478849:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320897.478851:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983185280:0@lo:38:kworker.0 00000100:00080000:3.0:1767320897.479047:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320897.479051:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320897.479054:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320897.479058:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6140 00000100:00080000:3.0:1767320897.479062:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320897.479065:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320897.479085:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595880 x1853163983185664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320897.479474:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983185664:0@lo:38:kworker.0 00000100:00100000:1.0:1767320897.479503:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.479574:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185664 00000100:00100000:3.0:1767320897.479578:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185664 00000100:00100000:3.0:1767320897.479599:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2635 00000100:00100000:3.0:1767320897.479602:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983185664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320897.479610:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320897.479639:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983185664:12345-0@lo:38:kworker.0 Request processed in 36us (136us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320897.479643:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2635 00000100:00080000:3.0:1767320897.479661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320897.479663:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320897.479665:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983185664:0@lo:38:kworker.0 00000100:00100000:0.0:1767320897.479831:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983184768:12345-0@lo:400:kworker.0 Request processed in 2172us (2620us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320897.479839:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1989 00010000:00080000:2.0:1767320897.479932:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638597800 x1853163983184768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320913 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320897.479944:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983184768:0@lo:400:kworker.0 00010000:00010000:3.0:1767320897.668614:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320897.988716:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983185792:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320897.988798:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320897.989178:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185792 00000100:00100000:3.0:1767320897.989187:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185792 00000100:00100000:3.0:1767320897.989250:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1328 00000100:00100000:3.0:1767320897.989260:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983185792:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320897.989313:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320897.989363:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983185792:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (564us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320897.989370:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1328 00000100:00100000:0.0:1767320897.989484:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983185792:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320897.992105:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983185920:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320897.992184:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320897.993748:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983185920 00000100:00100000:2.0:1767320897.993762:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983185920 00000100:00100000:2.0:1767320897.993858:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1329 00000100:00100000:2.0:1767320897.993870:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983185920:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320897.993903:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320897.996234:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983185920:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2358us (4041us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320897.996246:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1329 00000100:00100000:0.0:1767320897.996613:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983185920:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320898.696263:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320899.468367:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320899.469128:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320899.474739:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707520 00000100:00100000:2.0:1767320899.474864:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707520 00000100:00100000:2.0:1767320899.475017:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707392 00000100:00100000:2.0:1767320899.475021:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707392 00000100:00100000:2.0:1767320899.475060:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1991 00000100:00100000:2.0:1767320899.475073:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915707520:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320899.475194:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915707520:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 123us (6846us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320899.475204:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1991 00000100:00100000:2.0:1767320899.475227:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2636 00000100:00100000:2.0:1767320899.475232:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915707392:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320899.475974:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320899.476033:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915707392:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 800us (6912us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320899.476042:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2636 00000100:00080000:1.0:1767320899.476294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320899.717735:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320900.740794:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320901.765678:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320902.596579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320902.596595:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320902.596691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320902.596696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320902.596700:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320902.596711:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6146 00000100:00080000:1.0:1767320902.596717:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320902.596734:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320902.596765:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92680 x1853163983186176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320902.596820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320902.596824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320902.596826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320902.596829:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6146 00000100:00080000:1.0:1767320902.596832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320902.596836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320902.596846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91180 x1853163983186432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320902.596855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320902.596858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320902.596860:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320902.596864:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6146 00000100:00080000:1.0:1767320902.596867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320902.596870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320902.596885:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163983186560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320902.596920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320902.596925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320902.596927:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320902.596933:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6146 00000100:00100000:2.0:1767320902.596933:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983186176:0@lo:38:kworker.0 00000100:00080000:1.0:1767320902.596937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320902.596945:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320902.596957:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247100 x1853163983186944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320902.596987:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983186688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.597035:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320902.597042:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320902.597101:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983186048:0@lo:400:kworker.0 00000100:00100000:0.0:1767320902.597123:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983186816:0@lo:400:kworker.0 00000100:00100000:1.0:1767320902.597202:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320902.597209:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983186304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320902.597235:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.597265:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983186432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320902.597316:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186688 00000100:00100000:1.0:1767320902.597326:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186688 00000100:00100000:2.0:1767320902.597331:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.597357:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983186560:0@lo:38:kworker.0 00000100:00100000:2.0:1767320902.597388:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320902.597405:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1217 00000100:00100000:1.0:1767320902.597417:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983186688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.597431:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983186944:0@lo:38:kworker.0 00000100:00100000:1.0:1767320902.597479:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983186688:12345-0@lo:400:kworker.0 Request processed in 65us (439us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320902.597487:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1217 00000100:00100000:2.0:1767320902.597507:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320902.597516:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320902.597554:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186176 00000100:00100000:2.0:1767320902.597559:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186176 00000100:00100000:1.0:1767320902.597560:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983186688:0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.597589:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186304 00000100:00100000:2.0:1767320902.597594:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186304 00000100:00100000:1.0:1767320902.597595:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186432 00000100:00100000:1.0:1767320902.597601:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186432 00000100:00100000:1.0:1767320902.597619:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186944 00000100:00100000:1.0:1767320902.597624:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186944 00000100:00100000:2.0:1767320902.597633:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1992 00000100:00100000:2.0:1767320902.597643:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983186304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.597684:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983186304:12345-0@lo:400:kworker.0 Request processed in 43us (448us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320902.597752:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2637 00000100:00100000:1.0:1767320902.597758:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983186176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320902.597781:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320902.597815:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1992 00000100:00100000:1.0:1767320902.597822:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983186176:12345-0@lo:38:kworker.0 Request processed in 60us (786us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320902.597830:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2637 00000100:00100000:2.0:1767320902.597838:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2638 00000100:00100000:2.0:1767320902.597845:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983186432:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320902.597851:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2639 00000100:00100000:1.0:1767320902.597857:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983186944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320902.597860:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767320902.597870:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320902.597900:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983186432:12345-0@lo:38:kworker.0 Request processed in 52us (567us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320902.597910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983186944:12345-0@lo:38:kworker.0 Request processed in 49us (402us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320902.597910:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2638 00000100:00100000:1.0:1767320902.597919:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2639 00000100:00080000:1.0:1767320902.597943:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320902.597949:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320902.597956:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320902.597961:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983186944:0@lo:38:kworker.0 00010000:00080000:1.0:1767320902.597967:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d93480 x1853163983186304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320918 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320902.597975:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320902.597980:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983186304:0@lo:400:kworker.0 00000100:00080000:2.0:1767320902.597983:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320902.597990:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983186176:0@lo:38:kworker.0 00000100:00080000:2.0:1767320902.598003:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320902.598008:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320902.598012:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983186432:0@lo:38:kworker.0 00000100:00100000:1.0:1767320902.598076:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186560 00000100:00100000:1.0:1767320902.598082:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186560 00000100:00100000:1.0:1767320902.598106:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 684 00000100:00100000:1.0:1767320902.598112:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983186560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320902.598121:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320902.598154:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983186560:12345-0@lo:38:kworker.0 Request processed in 40us (766us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320902.598160:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 684 00000100:00080000:2.0:1767320902.599683:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320902.599699:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320902.599706:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983186560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320902.599880:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320902.600386:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186048 00000100:00100000:2.0:1767320902.600397:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186048 00000100:00080000:1.0:1767320902.600528:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320902.600581:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320902.600705:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983186816 00000100:00100000:1.0:1767320902.600710:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983186816 00000100:00100000:1.0:1767320902.600784:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1218 00000100:00100000:1.0:1767320902.600795:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983186816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320902.600856:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1993 00000100:00100000:2.0:1767320902.600868:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983186048:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320902.600884:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983186816:12345-0@lo:400:kworker.0 Request processed in 91us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320902.600892:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1218 00000100:00100000:2.0:1767320902.600929:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983186048:12345-0@lo:400:kworker.0 Request processed in 62us (1048us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320902.600937:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1993 00000100:00080000:0.0:1767320902.601003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320902.601034:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983186816:0@lo:400:kworker.0 00010000:00080000:3.0:1767320902.601322:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d90a80 x1853163983186048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320918 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320902.601334:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983186048:0@lo:400:kworker.0 00010000:00010000:2.0:1767320902.788649:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320903.109644:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320903.109679:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983187072:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320903.109797:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320903.109812:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320903.109827:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983187200:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320903.109866:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320903.109909:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187072 00000100:00100000:1.0:1767320903.109920:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187072 00000100:00100000:1.0:1767320903.109984:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187200 00000100:00100000:1.0:1767320903.109988:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187200 00000100:00100000:1.0:1767320903.110006:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1330 00000100:00100000:1.0:1767320903.110015:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983187072:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320903.110053:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320903.110107:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983187072:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (311us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320903.110115:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1330 00000100:00100000:1.0:1767320903.110135:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1331 00000100:00100000:1.0:1767320903.110140:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983187200:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320903.110158:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320903.110190:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983187200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320903.110196:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1331 00000100:00100000:1.0:1767320903.110246:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983187072:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320903.110421:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983187200:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320903.813807:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320904.588840:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320904.588890:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320904.589221:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707776 00000100:00100000:1.0:1767320904.589236:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707776 00000100:00080000:0.0:1767320904.589817:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320904.590884:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707648 00000100:00100000:2.0:1767320904.590898:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707648 00000100:00100000:2.0:1767320904.590971:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2640 00000100:00100000:2.0:1767320904.590982:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915707648:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320904.591011:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320904.591111:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915707648:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 129us (2219us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320904.591119:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2640 00000100:00100000:1.0:1767320904.591245:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1994 00000100:00100000:1.0:1767320904.591256:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915707776:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320904.591335:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915707776:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 81us (2510us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320904.591342:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1994 00010000:00010000:3.0:1767320904.842036:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320905.860781:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320906.884585:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320907.716535:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320907.716550:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767320907.716699:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983187328:0@lo:400:kworker.0 00000100:00100000:2.0:1767320907.716806:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.716935:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187328 00000100:00100000:1.0:1767320907.716953:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187328 00000100:00100000:1.0:1767320907.717032:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995 00000100:00100000:1.0:1767320907.717044:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983187328:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320907.717110:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767320907.717126:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320907.717134:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320907.717138:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320907.717146:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6151 00000100:00080000:0.0:1767320907.717151:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320907.717158:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320907.717187:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ebb80 x1853163983187456/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320907.717236:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320907.717240:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320907.717242:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320907.717245:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6151 00000100:00080000:0.0:1767320907.717248:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320907.717252:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320907.717262:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163983187712/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320907.717301:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320907.717305:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320907.717307:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320907.717310:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6151 00000100:00080000:0.0:1767320907.717313:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320907.717317:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320907.717328:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8000 x1853163983187840/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320907.717703:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983188096:0@lo:400:kworker.0 00000100:00100000:3.0:1767320907.717779:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320907.717847:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188096 00000100:00100000:3.0:1767320907.717856:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188096 00000100:00100000:1.0:1767320907.717913:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983187328:12345-0@lo:400:kworker.0 Request processed in 867us (1103us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320907.717929:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995 00000100:00100000:3.0:1767320907.717932:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1219 00000100:00100000:3.0:1767320907.717945:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983188096:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.717985:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983187584:0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.718027:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.718052:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320907.718057:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983187968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.718080:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.718107:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187968 00000100:00100000:1.0:1767320907.718113:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187968 00000100:00100000:1.0:1767320907.718150:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1220 00000100:00100000:1.0:1767320907.718159:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983187968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.718199:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983187968:12345-0@lo:400:kworker.0 Request processed in 41us (117us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320907.718211:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1220 00000100:00100000:1.0:1767320907.718258:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983187968:0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.718296:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187584 00000100:00100000:1.0:1767320907.718299:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187584 00000100:00100000:1.0:1767320907.718324:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1996 00000100:00100000:1.0:1767320907.718330:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983187584:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.718361:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983187584:12345-0@lo:400:kworker.0 Request processed in 31us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320907.718367:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1996 00010000:00080000:1.0:1767320907.718390:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9880 x1853163983187584/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320923 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320907.718401:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983187584:0@lo:400:kworker.0 00000100:00080000:1.0:1767320907.718447:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320907.718453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320907.718455:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320907.718469:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983187456:0@lo:38:kworker.0 00000100:00100000:1.0:1767320907.718499:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.718543:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983187712:0@lo:38:kworker.0 00000100:00100000:3.0:1767320907.718667:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983188096:12345-0@lo:400:kworker.0 Request processed in 721us (884us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320907.718677:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1219 00000100:00100000:3.0:1767320907.718708:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187456 00000100:00100000:3.0:1767320907.718713:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187456 00000100:00100000:3.0:1767320907.718737:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2641 00000100:00100000:3.0:1767320907.718742:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983187456:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320907.718761:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1767320907.718834:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320907.718842:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320907.718845:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320907.718850:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6151 00000100:00080000:0.0:1767320907.718854:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320907.718859:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320907.718888:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9880 x1853163983188224/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320907.718987:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983188096:0@lo:400:kworker.0 00000100:00100000:1.0:1767320907.719196:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.719235:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983187840:0@lo:38:kworker.0 00000100:00100000:1.0:1767320907.719265:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.719317:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983188224:0@lo:38:kworker.0 00000100:00100000:1.0:1767320907.719338:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320907.719362:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187712 00000100:00100000:1.0:1767320907.719367:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187712 00000100:00100000:1.0:1767320907.719387:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188224 00000100:00100000:1.0:1767320907.719389:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188224 00000100:00100000:1.0:1767320907.719404:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2642 00000100:00100000:1.0:1767320907.719409:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983187712:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320907.719426:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320907.719463:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983187712:12345-0@lo:38:kworker.0 Request processed in 52us (266us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320907.719469:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2642 00000100:00100000:1.0:1767320907.719484:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2643 00000100:00100000:1.0:1767320907.719487:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983188224:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320907.719497:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320907.719518:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983188224:12345-0@lo:38:kworker.0 Request processed in 29us (179us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320907.719523:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2643 00000100:00080000:1.0:1767320907.719552:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320907.719557:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320907.719560:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983188224:0@lo:38:kworker.0 00000100:00080000:1.0:1767320907.719568:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1767320907.719814:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983187840 00000100:00100000:0.0:1767320907.719819:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983187840 00000100:00100000:0.0:1767320907.719844:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 685 00000100:00100000:0.0:1767320907.719849:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983187840:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320907.719859:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320907.719893:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983187840:12345-0@lo:38:kworker.0 Request processed in 43us (627us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320907.719898:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 685 00000100:00080000:1.0:1767320907.720038:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320907.720042:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983187712:0@lo:38:kworker.0 00000100:00080000:1.0:1767320907.720057:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320907.720060:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320907.720063:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983187840:0@lo:38:kworker.0 00010000:00080000:2.0:1767320907.720159:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ead80 x1853163983187328/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320923 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320907.720172:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983187328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320907.721408:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983187456:12345-0@lo:38:kworker.0 Request processed in 2656us (2902us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320907.721423:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2641 00000100:00080000:1.0:1767320907.721478:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320907.721491:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320907.721496:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983187456:0@lo:38:kworker.0 00010000:00010000:3.0:1767320907.908887:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320908.229294:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320908.229326:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983188352:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320908.229427:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320908.229730:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320908.229743:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983188480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320908.229777:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320908.229866:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188352 00000100:00100000:1.0:1767320908.229878:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188352 00000100:00100000:1.0:1767320908.229950:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188480 00000100:00100000:1.0:1767320908.229954:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188480 00000100:00100000:1.0:1767320908.230155:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1332 00000100:00100000:1.0:1767320908.230169:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983188352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320908.230212:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320908.230309:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983188352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 142us (881us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320908.230317:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1332 00000100:00100000:1.0:1767320908.230339:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1333 00000100:00100000:1.0:1767320908.230346:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983188480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320908.230365:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320908.230400:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983188480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (621us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320908.230405:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1333 00000100:00100000:1.0:1767320908.230453:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983188352:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320908.230668:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983188480:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767320908.936906:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320909.707839:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320909.707908:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915707904 00000100:00100000:0.0:1767320909.707922:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915707904 00000100:00100000:0.0:1767320909.708108:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2644 00000100:00100000:0.0:1767320909.708119:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915707904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320909.708150:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320909.708236:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915707904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 117us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320909.708245:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2644 00000100:00100000:3.0:1767320909.708518:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320909.708755:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708032 00000100:00100000:0.0:1767320909.708761:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708032 00000100:00080000:1.0:1767320909.708812:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320909.708830:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 1997 00000100:00100000:0.0:1767320909.708839:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915708032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320909.708892:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915708032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 56us (374us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320909.708901:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 1997 00010000:00010000:3.0:1767320909.957096:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320910.980837:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320912.006093:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320912.837872:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320912.837887:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320912.837964:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320912.837970:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320912.837974:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320912.837984:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6156 00000100:00080000:3.0:1767320912.837990:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320912.837998:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320912.838024:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163983188736/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320912.838071:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320912.838075:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320912.838077:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320912.838081:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6156 00000100:00080000:3.0:1767320912.838083:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320912.838088:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320912.838100:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638596a00 x1853163983188992/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320912.838109:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320912.838113:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320912.838114:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320912.838118:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6156 00000100:00080000:3.0:1767320912.838121:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320912.838125:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320912.838136:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597480 x1853163983189120/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320912.838163:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320912.838166:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320912.838168:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320912.838171:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6156 00000100:00080000:3.0:1767320912.838174:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320912.838177:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320912.838188:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a581880 x1853163983189504/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320912.838364:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983189376:0@lo:400:kworker.0 00000100:00100000:2.0:1767320912.838461:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320912.838646:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320912.838652:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983188608:0@lo:400:kworker.0 00000100:00100000:2.0:1767320912.838679:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320912.838720:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188608 00000100:00100000:2.0:1767320912.838729:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188608 00000100:00100000:2.0:1767320912.838795:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1998 00000100:00100000:2.0:1767320912.838807:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983188608:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320912.838865:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983188608:12345-0@lo:400:kworker.0 Request processed in 60us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320912.838872:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1998 00000100:00080000:2.0:1767320912.838895:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767320912.838924:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638596300 x1853163983188608/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320928 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320912.838939:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983188608:0@lo:400:kworker.0 00000100:00100000:3.0:1767320912.839023:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189376 00000100:00100000:3.0:1767320912.839027:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189376 00000100:00100000:3.0:1767320912.839071:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1221 00000100:00100000:1.0:1767320912.839079:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983188864:0@lo:400:kworker.0 00000100:00100000:3.0:1767320912.839079:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983189376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320912.839110:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320912.839116:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983189376:12345-0@lo:400:kworker.0 Request processed in 39us (656us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320912.839122:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1221 00000100:00100000:1.0:1767320912.839129:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320912.839134:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983189248:0@lo:400:kworker.0 00000100:00100000:1.0:1767320912.839155:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320912.839161:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983188736:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.839178:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189248 00000100:00100000:1.0:1767320912.839182:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189248 00000100:00100000:3.0:1767320912.839185:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320912.839208:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1222 00000100:00100000:3.0:1767320912.839208:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983188992:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.839214:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983189248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320912.839228:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320912.839243:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983189248:12345-0@lo:400:kworker.0 Request processed in 29us (87us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320912.839244:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983189120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.839250:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1222 00000100:00100000:3.0:1767320912.839266:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320912.839298:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188864 00000100:00100000:1.0:1767320912.839302:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188864 00000100:00100000:3.0:1767320912.839312:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983189504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.839326:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188736 00000100:00100000:1.0:1767320912.839329:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188736 00000100:00100000:3.0:1767320912.839335:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320912.839338:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983188992 00000100:00100000:1.0:1767320912.839340:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983188992 00000100:00100000:1.0:1767320912.839349:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189504 00000100:00100000:1.0:1767320912.839351:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189504 00000100:00100000:3.0:1767320912.839360:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1999 00000100:00080000:2.0:1767320912.839456:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320912.839460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320912.839462:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320912.840396:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983188864:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320912.840471:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983188864:12345-0@lo:400:kworker.0 Request processed in 1103us (1359us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320912.840893:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2645 00000100:00100000:1.0:1767320912.840904:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983188736:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320912.840931:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320912.840990:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983188736:12345-0@lo:38:kworker.0 Request processed in 87us (1802us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320912.840998:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2645 00000100:00100000:1.0:1767320912.841032:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2646 00000100:00100000:1.0:1767320912.841038:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983188992:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320912.841048:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320912.841073:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983188992:12345-0@lo:38:kworker.0 Request processed in 34us (1844us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320912.841080:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2646 00000100:00100000:1.0:1767320912.841092:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2647 00000100:00100000:1.0:1767320912.841097:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983189504:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320912.841108:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320912.841132:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983189504:12345-0@lo:38:kworker.0 Request processed in 34us (1796us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320912.841139:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2647 00000100:00100000:1.0:1767320912.841200:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983189248:0@lo:400:kworker.0 00010000:00080000:1.0:1767320912.841216:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594700 x1853163983188864/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320928 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320912.841227:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983188864:0@lo:400:kworker.0 00000100:00080000:1.0:1767320912.841290:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320912.841296:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320912.841300:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983188736:0@lo:38:kworker.0 00000100:00080000:1.0:1767320912.841311:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320912.841315:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320912.841318:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983188992:0@lo:38:kworker.0 00000100:00080000:1.0:1767320912.841328:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320912.841332:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320912.841334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983189504:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.841381:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189120 00000100:00100000:1.0:1767320912.841389:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189120 00000100:00100000:1.0:1767320912.841421:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 686 00000100:00100000:1.0:1767320912.841426:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983189120:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320912.841437:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320912.841471:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983189120:12345-0@lo:38:kworker.0 Request processed in 43us (2202us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320912.841478:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 686 00000100:00080000:1.0:1767320912.841503:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320912.841507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320912.841511:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983189120:0@lo:38:kworker.0 00000100:00100000:1.0:1767320912.841576:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983189376:0@lo:400:kworker.0 00000100:00100000:3.0:1767320912.847335:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1999 00010000:00010000:3.0:1767320913.028497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320913.349403:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983189632:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320913.349523:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320913.349817:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189632 00000100:00100000:0.0:1767320913.349828:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189632 00000100:00100000:0.0:1767320913.349909:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1334 00000100:00100000:0.0:1767320913.349920:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983189632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320913.349963:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320913.350031:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983189632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (507us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320913.350040:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1334 00000100:00100000:3.0:1767320913.350344:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983189632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320913.352644:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320913.352674:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983189760:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320913.352766:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320913.353033:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189760 00000100:00100000:0.0:1767320913.353045:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189760 00000100:00100000:0.0:1767320913.353113:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1335 00000100:00100000:0.0:1767320913.353123:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983189760:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320913.353162:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320913.353221:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983189760:12345-0@lo:13:osp-pre-0-1.0 Request processed in 99us (455us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320913.353230:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1335 00000100:00100000:2.0:1767320913.353334:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983189760:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320913.810379:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767320913.810414:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915708160 offset 0 length 520: 4 00000100:00100000:1.0:1767320913.810453:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320913.810549:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708416 00000100:00100000:3.0:1767320913.810559:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708416 00000100:00100000:3.0:1767320913.810623:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708288 00000100:00100000:3.0:1767320913.810626:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708288 00000100:00100000:3.0:1767320913.810647:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2000 00000100:00100000:3.0:1767320913.810657:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915708416:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320913.810736:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915708416:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (367us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320913.810744:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2000 00000100:00100000:3.0:1767320913.810763:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2648 00000100:00100000:3.0:1767320913.810767:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915708288:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320913.810786:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320913.810812:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915708288:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (359us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320913.810819:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2648 00000100:00080000:1.0:1767320913.811631:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320914.052634:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320915.076492:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320916.100875:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320917.124656:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320917.956581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320917.956590:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320917.956643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320917.956646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320917.956650:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320917.956656:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6161 00000100:00080000:1.0:1767320917.956660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320917.956665:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320917.956681:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e245180 x1853163983190016/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320917.956701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320917.956703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320917.956704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320917.956706:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6161 00000100:00080000:1.0:1767320917.956709:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320917.956711:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320917.956718:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246a00 x1853163983190272/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320917.956724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320917.956725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320917.956726:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320917.956728:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6161 00000100:00080000:1.0:1767320917.956730:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320917.956732:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320917.956739:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247100 x1853163983190400/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320917.956751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320917.956754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320917.956763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320917.956765:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6161 00000100:00080000:1.0:1767320917.956767:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320917.956769:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320917.956776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90a80 x1853163983190784/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320917.956799:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983190144:0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.956857:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.956904:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320917.956908:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983190528:0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.956925:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.956965:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190528 00000100:00100000:1.0:1767320917.956971:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190528 00000100:00100000:1.0:1767320917.957024:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1223 00000100:00100000:1.0:1767320917.957032:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983190528:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.957076:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983190528:12345-0@lo:400:kworker.0 Request processed in 45us (148us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.957081:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1223 00000100:00080000:1.0:1767320917.957099:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320917.957128:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983190528:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.957188:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983190656:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.957219:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320917.957243:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320917.957249:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983189888:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.957268:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320917.957394:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190144 00000100:00100000:0.0:1767320917.957397:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190144 00000100:00100000:0.0:1767320917.957429:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983189888 00000100:00100000:0.0:1767320917.957431:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983189888 00000100:00100000:0.0:1767320917.957459:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2001 00000100:00100000:0.0:1767320917.957466:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983190144:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320917.957509:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983190144:12345-0@lo:400:kworker.0 Request processed in 44us (651us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.957511:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190656 00000100:00100000:0.0:1767320917.957516:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2001 00000100:00100000:1.0:1767320917.957516:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190656 00000100:00100000:0.0:1767320917.957527:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2002 00000100:00100000:0.0:1767320917.957533:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983189888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320917.957539:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1224 00000100:00100000:1.0:1767320917.957543:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983190656:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320917.957562:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983189888:12345-0@lo:400:kworker.0 Request processed in 29us (293us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320917.957568:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2002 00000100:00100000:1.0:1767320917.957569:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983190656:12345-0@lo:400:kworker.0 Request processed in 26us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320917.957574:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1224 00010000:00080000:0.0:1767320917.957599:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e245880 x1853163983190144/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320933 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320917.957609:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983190144:0@lo:400:kworker.0 00000100:00080000:3.0:1767320917.957664:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320917.957668:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320917.957685:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e246680 x1853163983189888/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320933 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320917.957693:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983189888:0@lo:400:kworker.0 00000100:00100000:3.0:1767320917.957702:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a581880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983190656:0@lo:400:kworker.0 00000100:00100000:2.0:1767320917.957782:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320892/real 1767320892] req@ffff93460a582300 x1853163983183360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320917 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320917.957797:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93460a582300 x1853163983183360/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320917 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320917.957810:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983190016:0@lo:38:kworker.0 00000100:00100000:2.0:1767320917.957841:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320917.957875:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983190272:0@lo:38:kworker.0 00000100:00100000:2.0:1767320917.957893:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320917.957910:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983190400:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.957931:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190016 00000100:00100000:2.0:1767320917.957932:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.957946:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190016 00000100:00100000:2.0:1767320917.957960:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983190784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.957961:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190272 00000100:00100000:1.0:1767320917.957973:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190272 00000100:00100000:2.0:1767320917.957979:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320917.957987:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190784 00000100:00100000:1.0:1767320917.957989:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190784 00000100:00080000:2.0:1767320917.957992:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320917.957998:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2649 00000100:00100000:1.0:1767320917.958003:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190016:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767320917.958004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320917.958007:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983183360:0@lo:250:kworker.0 00010000:00020000:1.0:1767320917.958019:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320917.958024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320917.958028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767320917.958030:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320917.958033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6140 00000100:00080000:2.0:1767320917.958036:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320917.958041:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320917.958052:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190016:12345-0@lo:38:kworker.0 Request processed in 49us (211us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320917.958056:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594a80 x1853163983190912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320917.958059:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2649 00000100:00100000:1.0:1767320917.958070:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2650 00000100:00100000:1.0:1767320917.958074:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190272:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320917.958077:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983190912:0@lo:250:kworker.0 00010000:00020000:1.0:1767320917.958082:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000400:00000100:2.0:1767320917.958094:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983190912 offset 0 length 520: 4 00000100:00100000:1.0:1767320917.958102:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190272:12345-0@lo:38:kworker.0 Request processed in 27us (208us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320917.958107:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2650 00000100:00080000:2.0:1767320917.958115:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320917.958117:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2651 00000100:00080000:2.0:1767320917.958118:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320917.958121:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190784:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320917.958122:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983190016:0@lo:38:kworker.0 00000100:00080000:2.0:1767320917.958128:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00020000:1.0:1767320917.958129:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767320917.958131:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320917.958133:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983190272:0@lo:38:kworker.0 00000100:00080000:2.0:1767320917.958149:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767320917.958150:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983190784:12345-0@lo:38:kworker.0 Request processed in 28us (169us total) trans 0 rc -19/-19 00000100:00080000:2.0:1767320917.958152:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320917.958154:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983190784:0@lo:38:kworker.0 00000100:00100000:1.0:1767320917.958156:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2651 00000100:00100000:3.0:1767320917.958218:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983190400 00000100:00100000:3.0:1767320917.958222:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983190400 00000100:00100000:3.0:1767320917.958247:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 687 00000100:00100000:3.0:1767320917.958252:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983190400:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320917.958262:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320917.958316:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983190400:12345-0@lo:38:kworker.0 Request processed in 62us (383us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320917.958322:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 687 00000100:00080000:2.0:1767320917.958425:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320917.958430:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320917.958433:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983190400:0@lo:38:kworker.0 00010000:00010000:2.0:1767320918.154849:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320918.468457:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983191040:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320918.468467:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320918.468484:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983191168:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320918.476777:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320918.476999:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191168 00000100:00100000:3.0:1767320918.477011:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191168 00000100:00100000:3.0:1767320918.477078:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1336 00000100:00100000:3.0:1767320918.477088:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983191168:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320918.477121:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320918.477193:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983191168:12345-0@lo:13:osp-pre-0-1.0 Request processed in 107us (425us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320918.477201:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1336 00000100:00100000:2.0:1767320918.477315:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983191168:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320918.479094:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320918.481821:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191040 00000100:00100000:0.0:1767320918.481838:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191040 00000100:00100000:0.0:1767320918.481918:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1337 00000100:00100000:0.0:1767320918.481931:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983191040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320918.481965:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320918.482053:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983191040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (2963us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320918.482061:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1337 00000100:00100000:1.0:1767320918.482172:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983191040:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320918.925778:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320918.925868:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320918.926126:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708672 00000100:00100000:2.0:1767320918.926135:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708672 00000100:00100000:2.0:1767320918.926192:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708544 00000100:00100000:2.0:1767320918.926196:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708544 00000100:00100000:2.0:1767320918.926219:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2003 00000100:00100000:2.0:1767320918.926230:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915708672:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767320918.926341:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320918.926348:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915708672:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 119us (580us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320918.926358:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2003 00000100:00100000:3.0:1767320918.926371:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2652 00000100:00100000:3.0:1767320918.926378:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915708544:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320918.926398:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320918.926438:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915708544:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (568us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320918.926445:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2652 00010000:00010000:3.0:1767320919.172673:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320920.196657:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320921.221447:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320922.244720:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320923.076551:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320923.076563:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320923.076640:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320923.076646:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320923.076649:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320923.076683:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6166 00000100:00080000:0.0:1767320923.076689:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320923.076696:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320923.076720:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163983191424/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320923.076742:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320923.076760:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983191296:0@lo:400:kworker.0 00000100:00080000:0.0:1767320923.076799:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320923.076804:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320923.076807:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320923.076811:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6166 00000100:00080000:0.0:1767320923.076814:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320923.076818:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320923.076833:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ebb80 x1853163983191680/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320923.076836:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320923.076849:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320923.076852:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320923.076854:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320923.076859:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6166 00000100:00080000:0.0:1767320923.076861:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320923.076883:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320923.076890:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191296 00000100:00100000:0.0:1767320923.076897:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea680 x1853163983191808/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320923.076898:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191296 00000100:00080000:0.0:1767320923.076935:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320923.076939:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320923.076941:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320923.076945:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6166 00000100:00080000:0.0:1767320923.076948:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767320923.076960:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004 00000100:00080000:0.0:1767320923.076966:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320923.076973:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983191296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320923.076981:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246a00 x1853163983192192/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320923.077008:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983191552:0@lo:400:kworker.0 00000100:00100000:3.0:1767320923.077025:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983191296:12345-0@lo:400:kworker.0 Request processed in 55us (191us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320923.077033:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1767320923.077046:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320923.077104:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983191424:0@lo:38:kworker.0 00000100:00100000:2.0:1767320923.077172:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983191936:0@lo:400:kworker.0 00000100:00100000:1.0:1767320923.077237:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320923.086745:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320923.086849:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191936 00000100:00100000:2.0:1767320923.086859:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191936 00000100:00100000:2.0:1767320923.086920:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1225 00000100:00100000:2.0:1767320923.086931:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983191936:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320923.087017:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983191936:12345-0@lo:400:kworker.0 Request processed in 88us (294us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320923.087025:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1225 00000100:00100000:2.0:1767320923.087049:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320923.087060:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983192064:0@lo:400:kworker.0 00000100:00100000:2.0:1767320923.087094:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320923.087116:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192064 00000100:00100000:2.0:1767320923.087120:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192064 00000100:00100000:2.0:1767320923.087156:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1226 00000100:00100000:2.0:1767320923.087163:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983192064:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320923.088670:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983192064:12345-0@lo:400:kworker.0 Request processed in 34us (102us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320923.088684:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1226 00000100:00100000:1.0:1767320923.088797:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983191680:0@lo:38:kworker.0 00000100:00100000:1.0:1767320923.088850:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320923.093439:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191552 00000100:00100000:3.0:1767320923.093458:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191552 00000100:00100000:3.0:1767320923.095307:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191424 00000100:00100000:3.0:1767320923.095320:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191424 00000100:00100000:3.0:1767320923.095360:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2005 00000100:00100000:3.0:1767320923.095373:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983191552:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320923.095472:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983191552:12345-0@lo:400:kworker.0 Request processed in 103us (18426us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320923.095479:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2005 00000100:00100000:3.0:1767320923.095501:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2653 00000100:00100000:3.0:1767320923.095505:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983191424:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320923.095539:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320923.095568:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983191424:12345-0@lo:38:kworker.0 Request processed in 62us (18331us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320923.095574:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2653 00000100:00080000:3.0:1767320923.095598:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767320923.095603:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320923.095606:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320923.095741:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983191936:0@lo:400:kworker.0 00000100:00100000:3.0:1767320923.095791:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983192064:0@lo:400:kworker.0 00010000:00080000:3.0:1767320923.095831:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e9880 x1853163983191552/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320939 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320923.095841:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983191552:0@lo:400:kworker.0 00010000:00080000:3.0:1767320923.095870:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395e8000 x1853163983191296/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320939 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320923.095879:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983191296:0@lo:400:kworker.0 00000100:00100000:1.0:1767320923.096802:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983191808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320923.096848:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320923.096880:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191808 00000100:00100000:1.0:1767320923.096887:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191808 00000100:00100000:1.0:1767320923.096919:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 688 00000100:00100000:1.0:1767320923.096926:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983191808:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320923.096942:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320923.097516:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983191680 00000100:00100000:3.0:1767320923.097523:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983191680 00000100:00100000:3.0:1767320923.097555:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2654 00000100:00100000:3.0:1767320923.097560:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983191680:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320923.097573:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320923.097616:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983191680:12345-0@lo:38:kworker.0 Request processed in 54us (8765us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320923.097623:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2654 00000100:00100000:1.0:1767320923.097767:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983191808:12345-0@lo:38:kworker.0 Request processed in 839us (917us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320923.097774:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 688 00000100:00080000:1.0:1767320923.097814:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320923.097819:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320923.097823:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983191808:0@lo:38:kworker.0 00000100:00100000:1.0:1767320923.097831:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983192192:0@lo:38:kworker.0 00000100:00100000:1.0:1767320923.097859:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320923.097892:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320923.097896:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320923.097899:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983191424:0@lo:38:kworker.0 00000100:00080000:1.0:1767320923.097907:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320923.097910:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320923.097913:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983191680:0@lo:38:kworker.0 00000100:00100000:3.0:1767320923.098482:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192192 00000100:00100000:3.0:1767320923.098490:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192192 00000100:00100000:3.0:1767320923.098531:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2655 00000100:00100000:3.0:1767320923.098538:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983192192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320923.098553:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320923.098596:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a582300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983192192:12345-0@lo:38:kworker.0 Request processed in 57us (736us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320923.098602:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2655 00000100:00080000:1.0:1767320923.100661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320923.100674:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320923.100679:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983192192:0@lo:38:kworker.0 00010000:00010000:3.0:1767320923.268630:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320923.588605:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983192320:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320923.588687:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320923.588874:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192320 00000100:00100000:0.0:1767320923.588888:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192320 00000100:00100000:0.0:1767320923.588955:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1338 00000100:00100000:0.0:1767320923.588966:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983192320:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320923.589001:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320923.589108:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983192320:12345-0@lo:13:osp-pre-0-1.0 Request processed in 143us (420us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320923.589117:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1338 00000100:00100000:1.0:1767320923.590886:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320923.590908:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983192448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320923.590977:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320923.591111:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192448 00000100:00100000:2.0:1767320923.591119:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192448 00000100:00100000:2.0:1767320923.591168:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1339 00000100:00100000:2.0:1767320923.591176:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983192448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320923.591205:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320923.591257:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983192448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320923.591264:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1339 00000100:00100000:1.0:1767320923.591480:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983192448:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320923.597953:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983192320:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320924.043939:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320924.044067:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708800 00000100:00100000:1.0:1767320924.044077:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708800 00000100:00100000:1.0:1767320924.044133:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2656 00000100:00100000:1.0:1767320924.044142:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915708800:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320924.044168:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320924.044245:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915708800:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 103us (317us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320924.044253:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2656 00000100:00100000:2.0:1767320924.046199:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320924.046541:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915708928 00000100:00100000:0.0:1767320924.046551:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915708928 00000100:00100000:0.0:1767320924.046666:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2006 00000100:00100000:0.0:1767320924.046687:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915708928:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320924.046812:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915708928:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 135us (618us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320924.046820:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2006 00000100:00080000:3.0:1767320924.047075:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320924.292504:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320925.318439:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320926.341833:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320927.366485:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320928.197835:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320928.197845:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320928.197914:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320928.197919:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320928.197922:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320928.197929:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6171 00000100:00080000:0.0:1767320928.197933:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320928.197939:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320928.197961:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246680 x1853163983192704/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320928.198015:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320928.198019:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320928.198020:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320928.198024:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6171 00000100:00080000:0.0:1767320928.198026:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320928.198030:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320928.198040:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247100 x1853163983192960/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320928.198049:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320928.198051:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320928.198053:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320928.198056:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6171 00000100:00080000:0.0:1767320928.198058:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320928.198062:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320928.198073:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5f80 x1853163983193088/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320928.198092:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320928.198095:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320928.198097:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320928.198100:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6171 00000100:00080000:0.0:1767320928.198102:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320928.198105:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320928.198114:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5880 x1853163983193472/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320928.198154:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983192832:0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.198294:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983192576:0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.198343:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320928.198382:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767320928.198389:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983193344:0@lo:400:kworker.0 00000100:00100000:2.0:1767320928.198406:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983192704:0@lo:38:kworker.0 00000100:00100000:1.0:1767320928.198430:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320928.199175:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983193216:0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.202175:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192576 00000100:00100000:1.0:1767320928.202289:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192576 00000100:00100000:1.0:1767320928.202354:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007 00000100:00100000:1.0:1767320928.202362:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983192576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.202498:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a583480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983192576:12345-0@lo:400:kworker.0 Request processed in 137us (4155us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320928.202504:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007 00010000:00080000:1.0:1767320928.202542:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e245880 x1853163983192576/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320944 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320928.202553:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e245880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983192576:0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.202583:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193344 00000100:00100000:1.0:1767320928.202586:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193344 00000100:00100000:1.0:1767320928.202606:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1227 00000100:00100000:1.0:1767320928.202610:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983193344:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.202633:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983193344:12345-0@lo:400:kworker.0 Request processed in 24us (4205us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320928.202637:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1227 00000100:00100000:1.0:1767320928.202652:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983193344:0@lo:400:kworker.0 00000100:00080000:1.0:1767320928.202678:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320928.202683:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320928.203762:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320928.204389:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193216 00000100:00100000:1.0:1767320928.204405:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193216 00000100:00100000:1.0:1767320928.204450:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1228 00000100:00100000:1.0:1767320928.204456:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983193216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.204500:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983193216:12345-0@lo:400:kworker.0 Request processed in 45us (742us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320928.204504:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1228 00000100:00080000:3.0:1767320928.204621:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320928.204646:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983193216:0@lo:400:kworker.0 00000100:00100000:2.0:1767320928.205925:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320928.205964:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983192960:0@lo:38:kworker.0 00000100:00100000:2.0:1767320928.206004:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320928.206083:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983193088:0@lo:38:kworker.0 00000100:00100000:2.0:1767320928.206110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320928.206150:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983193472:0@lo:38:kworker.0 00000100:00100000:2.0:1767320928.206262:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320928.206310:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192704 00000100:00100000:3.0:1767320928.206317:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192704 00000100:00100000:3.0:1767320928.206341:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192960 00000100:00100000:3.0:1767320928.206343:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192960 00000100:00100000:3.0:1767320928.206349:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193472 00000100:00100000:3.0:1767320928.206351:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193472 00000100:00100000:3.0:1767320928.206359:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2657 00000100:00100000:3.0:1767320928.206363:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983192704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320928.206379:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320928.206421:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983192704:12345-0@lo:38:kworker.0 Request processed in 57us (501us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320928.206425:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2657 00000100:00100000:3.0:1767320928.206436:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2658 00000100:00100000:3.0:1767320928.206438:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983192960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320928.206443:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320928.206456:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983192960:12345-0@lo:38:kworker.0 Request processed in 17us (452us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320928.206459:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2658 00000100:00100000:3.0:1767320928.206465:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2659 00000100:00100000:3.0:1767320928.206467:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983193472:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320928.206472:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320928.206482:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983193472:12345-0@lo:38:kworker.0 Request processed in 14us (233us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320928.206486:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2659 00000100:00100000:3.0:1767320928.206516:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193088 00000100:00100000:3.0:1767320928.206519:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193088 00000100:00100000:3.0:1767320928.206532:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 689 00000100:00100000:3.0:1767320928.206534:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983193088:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320928.206540:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320928.206555:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983193088:12345-0@lo:38:kworker.0 Request processed in 20us (445us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320928.206558:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 689 00000100:00080000:2.0:1767320928.206725:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320928.206731:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320928.206735:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983192704:0@lo:38:kworker.0 00000100:00080000:2.0:1767320928.206745:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320928.206748:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320928.206750:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983192960:0@lo:38:kworker.0 00000100:00080000:2.0:1767320928.206758:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320928.206761:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320928.206765:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983193088:0@lo:38:kworker.0 00000100:00080000:2.0:1767320928.206772:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320928.206775:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320928.206777:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983193472:0@lo:38:kworker.0 00000100:00100000:0.0:1767320928.208584:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320928.214540:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983192832 00000100:00100000:1.0:1767320928.214554:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983192832 00000100:00100000:1.0:1767320928.214626:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2008 00000100:00100000:1.0:1767320928.214633:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983192832:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320928.214728:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983192832:12345-0@lo:400:kworker.0 Request processed in 95us (6149us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320928.214737:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2008 00010000:00080000:2.0:1767320928.214940:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e246a00 x1853163983192832/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320944 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320928.214955:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983192832:0@lo:400:kworker.0 00000100:00080000:3.0:1767320928.215027:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320928.388505:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320928.708801:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320928.708826:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983193600:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320928.708898:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320928.709668:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193600 00000100:00100000:0.0:1767320928.709677:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193600 00000100:00100000:0.0:1767320928.709748:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1340 00000100:00100000:0.0:1767320928.709757:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983193600:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320928.709792:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320928.710000:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983193600:12345-0@lo:13:osp-pre-0-1.0 Request processed in 243us (1100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320928.710009:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1340 00000100:00100000:3.0:1767320928.711048:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320928.711071:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983193728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320928.711137:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320928.711675:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193728 00000100:00100000:0.0:1767320928.711692:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193728 00000100:00100000:0.0:1767320928.711744:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1341 00000100:00100000:0.0:1767320928.711756:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983193728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320928.711788:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320928.711838:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983193728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (701us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320928.711847:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1341 00000100:00100000:3.0:1767320928.712425:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983193728:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320928.713633:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983193600:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320929.163920:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320929.163962:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320929.164045:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709184 00000100:00100000:1.0:1767320929.164059:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709184 00000100:00100000:1.0:1767320929.164163:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709056 00000100:00100000:1.0:1767320929.164167:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709056 00000100:00100000:1.0:1767320929.164196:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2009 00000100:00100000:1.0:1767320929.164207:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915709184:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320929.164242:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2660 00000100:00100000:0.0:1767320929.164250:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709056:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:1.0:1767320929.164360:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915709184:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 154us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320929.164369:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2009 00010000:00020000:0.0:1767320929.164394:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320929.164432:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709056:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 180us (467us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320929.164439:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2660 00000100:00080000:3.0:1767320929.164550:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320929.412844:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320930.436719:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320931.462431:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320932.484641:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320933.316678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320933.316693:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320933.316778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320933.316784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320933.316789:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320933.316803:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6176 00000100:00080000:1.0:1767320933.316810:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320933.316820:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320933.316853:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469df80 x1853163983193984/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320933.316890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320933.316893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320933.316895:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320933.316900:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6176 00000100:00080000:1.0:1767320933.316902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320933.316907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320933.316919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f800 x1853163983194240/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320933.316928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320933.316932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767320933.316933:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983193856:0@lo:400:kworker.0 00000100:00080000:1.0:1767320933.316934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320933.316947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6176 00000100:00080000:1.0:1767320933.316953:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320933.316958:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320933.316976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c000 x1853163983194368/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320933.317006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320933.317010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320933.317012:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320933.317018:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6176 00000100:00080000:1.0:1767320933.317021:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320933.317026:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320933.317042:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320933.317044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c380 x1853163983194752/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320933.317080:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983194112:0@lo:400:kworker.0 00000100:00100000:3.0:1767320933.317132:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320933.317145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983194624:0@lo:400:kworker.0 00000100:00100000:1.0:1767320933.317151:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.317319:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983193984:0@lo:38:kworker.0 00000100:00100000:0.0:1767320933.317392:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193856 00000100:00100000:0.0:1767320933.317404:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193856 00000100:00100000:0.0:1767320933.317479:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194112 00000100:00100000:0.0:1767320933.317483:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194112 00000100:00100000:0.0:1767320933.317515:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010 00000100:00100000:0.0:1767320933.317527:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983193856:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320933.317833:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320933.317841:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983194496:0@lo:400:kworker.0 00000100:00100000:3.0:1767320933.320044:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320933.320175:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2011 00000100:00100000:3.0:1767320933.320192:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983194112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320933.320253:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983194112:12345-0@lo:400:kworker.0 Request processed in 69us (3107us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320933.320261:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2011 00000100:00080000:3.0:1767320933.320363:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320933.320396:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194624 00000100:00100000:3.0:1767320933.320404:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194624 00000100:00100000:3.0:1767320933.320449:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1229 00000100:00100000:3.0:1767320933.320455:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983194624:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320933.320494:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983194624:12345-0@lo:400:kworker.0 Request processed in 40us (458us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320933.320501:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1229 00000100:00080000:3.0:1767320933.320524:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320933.320558:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983194624:0@lo:400:kworker.0 00000100:00100000:2.0:1767320933.320630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.320687:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983194240:0@lo:38:kworker.0 00000100:00100000:2.0:1767320933.320722:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.320739:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983194368:0@lo:38:kworker.0 00000100:00100000:2.0:1767320933.320767:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.320791:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983194752:0@lo:38:kworker.0 00000100:00100000:2.0:1767320933.320812:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.320857:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194368 00000100:00100000:2.0:1767320933.320864:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194368 00000100:00100000:2.0:1767320933.320896:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 690 00000100:00100000:2.0:1767320933.320903:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983194368:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320933.320928:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320933.320982:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983194368:12345-0@lo:38:kworker.0 Request processed in 65us (200us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320933.320989:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 690 00000100:00100000:2.0:1767320933.321015:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983193984 00000100:00100000:2.0:1767320933.321019:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983193984 00000100:00100000:2.0:1767320933.321038:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194240 00000100:00100000:2.0:1767320933.321046:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194240 00000100:00100000:2.0:1767320933.321054:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194752 00000100:00100000:2.0:1767320933.321056:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194752 00000100:00100000:2.0:1767320933.321065:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2661 00000100:00100000:2.0:1767320933.321069:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983193984:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320933.321078:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320933.321105:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983193984:12345-0@lo:38:kworker.0 Request processed in 35us (481us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320933.321110:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2661 00000100:00100000:2.0:1767320933.321120:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2662 00000100:00100000:2.0:1767320933.321123:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983194240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320933.321132:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320933.321151:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983194240:12345-0@lo:38:kworker.0 Request processed in 27us (428us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320933.321169:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2662 00000100:00100000:2.0:1767320933.321180:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2663 00000100:00100000:2.0:1767320933.321184:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983194752:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320933.321193:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320933.321216:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983194752:12345-0@lo:38:kworker.0 Request processed in 30us (403us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320933.321222:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2663 00000100:00080000:2.0:1767320933.321298:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320933.321306:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320933.321310:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983193984:0@lo:38:kworker.0 00000100:00080000:2.0:1767320933.321321:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320933.321325:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320933.321327:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983194240:0@lo:38:kworker.0 00000100:00080000:2.0:1767320933.321335:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320933.321338:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320933.321341:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983194368:0@lo:38:kworker.0 00000100:00080000:2.0:1767320933.321347:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320933.321350:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320933.321352:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983194752:0@lo:38:kworker.0 00000100:00100000:0.0:1767320933.322180:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983193856:12345-0@lo:400:kworker.0 Request processed in 4647us (5134us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320933.322194:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010 00010000:00080000:3.0:1767320933.322505:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ce00 x1853163983193856/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320949 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320933.322522:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983193856:0@lo:400:kworker.0 00000100:00100000:1.0:1767320933.322644:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.322735:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194496 00000100:00100000:2.0:1767320933.322749:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194496 00010000:00080000:1.0:1767320933.322752:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469f480 x1853163983194112/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320949 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320933.322770:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983194112:0@lo:400:kworker.0 00000100:00100000:2.0:1767320933.322832:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1230 00000100:00100000:2.0:1767320933.322858:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983194496:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767320933.322893:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320933.322951:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983194496:12345-0@lo:400:kworker.0 Request processed in 109us (307us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320933.322961:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1230 00000100:00100000:1.0:1767320933.323092:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983194496:0@lo:400:kworker.0 00010000:00010000:2.0:1767320933.510868:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320933.828706:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320933.828744:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983194880:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320933.828842:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320933.829296:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983194880 00000100:00100000:1.0:1767320933.829311:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983194880 00000100:00100000:1.0:1767320933.829391:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1342 00000100:00100000:1.0:1767320933.829402:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983194880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320933.829442:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320933.829571:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983194880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 170us (729us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320933.829579:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1342 00000100:00100000:3.0:1767320933.830359:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767320933.830389:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983195008:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320933.830444:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320933.830840:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195008 00000100:00100000:2.0:1767320933.830850:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195008 00000100:00100000:2.0:1767320933.830915:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1343 00000100:00100000:2.0:1767320933.830925:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983195008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320933.830960:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320933.831018:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983195008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 95us (573us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320933.831025:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1343 00000100:00100000:2.0:1767320933.831083:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983195008:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320933.832099:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983194880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320934.292028:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320934.295364:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320934.295514:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709312 00000100:00100000:0.0:1767320934.295526:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709312 00000100:00100000:0.0:1767320934.295579:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709440 00000100:00100000:0.0:1767320934.295583:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709440 00000100:00100000:0.0:1767320934.295660:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2012 00000100:00100000:0.0:1767320934.295672:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915709440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320934.295775:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915709440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 104us (423us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320934.295784:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2012 00000100:00100000:0.0:1767320934.295804:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2664 00000100:00100000:0.0:1767320934.295809:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320934.296486:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320934.296519:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 708us (4972us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320934.296526:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2664 00000100:00080000:1.0:1767320934.297900:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320934.534316:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320935.556805:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320936.584415:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320937.605123:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767320938.437241:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320938.437255:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767320938.437376:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320938.437381:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320938.437385:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320938.437396:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6181 00000100:00080000:0.0:1767320938.437403:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320938.437412:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320938.437440:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee5180 x1853163983195264/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320938.437492:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767320938.437494:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983195264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320938.437496:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320938.437498:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320938.437502:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6181 00000100:00080000:0.0:1767320938.437505:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320938.437511:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320938.437522:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016300 x1853163983195520/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320938.437530:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320938.437533:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320938.437535:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320938.437539:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6181 00000100:00080000:0.0:1767320938.437541:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320938.437545:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320938.437557:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163983195648/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320938.437615:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320938.437618:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320938.437620:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320938.437624:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6181 00000100:00080000:0.0:1767320938.437626:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320938.437630:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320938.437643:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b480 x1853163983196032/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320938.437667:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983195392:0@lo:400:kworker.0 00000100:00100000:2.0:1767320938.437671:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320938.437758:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983195904:0@lo:400:kworker.0 00000100:00100000:1.0:1767320938.437837:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983195136:0@lo:400:kworker.0 00000100:00100000:0.0:1767320938.439666:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320938.439778:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767320938.439792:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983195776:0@lo:400:kworker.0 00000100:00100000:0.0:1767320938.439857:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320938.439915:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195264 00000100:00100000:0.0:1767320938.439925:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195264 00000100:00100000:0.0:1767320938.439976:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195392 00000100:00100000:0.0:1767320938.439981:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195392 00000100:00100000:0.0:1767320938.440091:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013 00000100:00100000:0.0:1767320938.440103:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983195392:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320938.440179:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983195392:12345-0@lo:400:kworker.0 Request processed in 78us (524us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320938.440187:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013 00000100:00100000:0.0:1767320938.440207:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2665 00000100:00100000:0.0:1767320938.440212:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983195264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320938.440236:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320938.440293:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983195264:12345-0@lo:38:kworker.0 Request processed in 56us (2599us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320938.440300:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2665 00000100:00100000:0.0:1767320938.440321:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195776 00000100:00100000:0.0:1767320938.440326:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195776 00000100:00100000:0.0:1767320938.440357:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1231 00000100:00100000:0.0:1767320938.440363:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983195776:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320938.440398:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983195776:12345-0@lo:400:kworker.0 Request processed in 35us (540us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320938.440404:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1231 00000100:00100000:0.0:1767320938.440459:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983195776:0@lo:400:kworker.0 00010000:00080000:0.0:1767320938.440473:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015500 x1853163983195392/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320954 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320938.440490:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983195392:0@lo:400:kworker.0 00000100:00080000:0.0:1767320938.440537:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320938.440542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320938.441629:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320938.441640:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320938.441645:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983195264:0@lo:38:kworker.0 00000100:00100000:2.0:1767320938.441840:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983195520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320938.441910:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320938.441961:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983195648:0@lo:38:kworker.0 00000100:00100000:0.0:1767320938.442073:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195520 00000100:00100000:0.0:1767320938.442082:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195520 00000100:00100000:0.0:1767320938.442128:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2666 00000100:00100000:0.0:1767320938.442136:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983195520:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320938.442162:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320938.442208:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983195520:12345-0@lo:38:kworker.0 Request processed in 71us (296us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320938.442214:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2666 00000100:00100000:2.0:1767320938.442692:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320938.442737:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983196032:0@lo:38:kworker.0 00000100:00100000:2.0:1767320938.442771:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320938.442809:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320938.442814:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320938.442818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983195520:0@lo:38:kworker.0 00000100:00100000:2.0:1767320938.442845:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196032 00000100:00100000:2.0:1767320938.442850:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196032 00000100:00100000:2.0:1767320938.442877:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2667 00000100:00100000:2.0:1767320938.442882:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983196032:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320938.442896:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320938.442929:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983196032:12345-0@lo:38:kworker.0 Request processed in 44us (156us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320938.442937:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2667 00000100:00080000:2.0:1767320938.442963:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320938.442968:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320938.442971:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983196032:0@lo:38:kworker.0 00000100:00100000:3.0:1767320938.447870:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320938.448977:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195648 00000100:00100000:0.0:1767320938.449007:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195648 00000100:00100000:0.0:1767320938.449085:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 691 00000100:00100000:0.0:1767320938.449097:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983195648:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320938.449132:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320938.449235:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983195648:12345-0@lo:38:kworker.0 Request processed in 139us (7240us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320938.449244:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 691 00000100:00100000:2.0:1767320938.449742:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195904 00000100:00100000:2.0:1767320938.449751:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195904 00000100:00100000:2.0:1767320938.449830:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1232 00000100:00100000:2.0:1767320938.449840:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983195904:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320938.449914:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983195904:12345-0@lo:400:kworker.0 Request processed in 75us (2059us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320938.449920:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1232 00000100:00080000:2.0:1767320938.449988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320938.449994:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320938.450009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983195648:0@lo:38:kworker.0 00000100:00100000:3.0:1767320938.450175:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983195904:0@lo:400:kworker.0 00000100:00080000:0.0:1767320938.450298:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320938.450671:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320938.450832:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983195136 00000100:00100000:2.0:1767320938.450839:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983195136 00000100:00100000:2.0:1767320938.450962:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2014 00000100:00100000:2.0:1767320938.450972:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983195136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320938.451049:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983195136:12345-0@lo:400:kworker.0 Request processed in 78us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320938.451057:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2014 00010000:00080000:1.0:1767320938.451316:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee6300 x1853163983195136/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320954 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320938.451330:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983195136:0@lo:400:kworker.0 00000100:00080000:0.0:1767320938.452328:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320938.630355:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320938.948633:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983196160:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320938.948732:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320938.948913:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196160 00000100:00100000:2.0:1767320938.948925:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196160 00000100:00100000:0.0:1767320938.949252:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983196288:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320938.949325:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320938.949455:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196288 00000100:00100000:1.0:1767320938.949461:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196288 00000100:00100000:1.0:1767320938.949529:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1344 00000100:00100000:1.0:1767320938.949540:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983196288:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320938.949580:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320938.949635:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983196288:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320938.949643:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1344 00000100:00100000:0.0:1767320938.949749:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983196288:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320938.954455:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1345 00000100:00100000:2.0:1767320938.954472:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983196160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320938.954540:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320938.954669:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983196160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 199us (5936us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320938.954679:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1345 00000100:00100000:1.0:1767320938.955559:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983196160:0@lo:13:osp-pre-1-1.0 00000400:00000100:2.0:1767320939.415914:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915709568 offset 0 length 520: 4 00000100:00100000:2.0:1767320939.415978:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320939.416062:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709696 00000100:00100000:3.0:1767320939.416076:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709696 00000100:00100000:3.0:1767320939.416422:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2668 00000100:00100000:3.0:1767320939.416433:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709696:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320939.416462:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320939.416551:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915709696:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 119us (572us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320939.416637:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320939.416724:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709824 00000100:00100000:0.0:1767320939.416729:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709824 00000100:00080000:1.0:1767320939.416863:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320939.423302:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2668 00000100:00100000:0.0:1767320939.423567:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2015 00000100:00100000:0.0:1767320939.423603:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915709824:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320939.424062:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915709824:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 481us (7423us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320939.424071:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2015 00010000:00010000:2.0:1767320939.656783:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320940.679364:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320941.704736:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320942.532609:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320917/real 1767320917] req@ffff934638594a80 x1853163983190912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320942 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320942.532656:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934638594a80 x1853163983190912/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320942 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320942.532682:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.532715:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320942.532719:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983190912:0@lo:250:kworker.0 00000100:00080000:0.0:1767320942.533476:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.533484:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320942.533488:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.533494:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6166 00000100:00080000:0.0:1767320942.533500:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.533508:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320942.533558:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190aa00 x1853163983196416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.533645:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.533649:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.533651:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.533655:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6186 00000100:00080000:0.0:1767320942.533658:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.533663:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320942.533681:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909f80 x1853163983196672/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.533716:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.533720:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.533721:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.533726:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6186 00000100:00080000:0.0:1767320942.533728:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.533732:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320942.533741:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631908000 x1853163983196928/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.533755:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.533758:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.533760:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.533764:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6186 00000100:00080000:0.0:1767320942.533766:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.533769:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320942.533778:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463190b800 x1853163983197056/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767320942.533797:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320942.533800:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767320942.533802:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320942.533805:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6186 00000100:00080000:0.0:1767320942.533808:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320942.533811:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320942.533820:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631908e00 x1853163983197440/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320942.533854:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983196800:0@lo:400:kworker.0 00000100:00100000:0.0:1767320942.533935:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320942.534016:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320942.534024:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983197184:0@lo:400:kworker.0 00000100:00100000:0.0:1767320942.534052:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320942.534085:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196800 00000100:00100000:0.0:1767320942.534094:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196800 00000100:00100000:0.0:1767320942.534157:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016 00000100:00100000:0.0:1767320942.534165:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983196800:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320942.534214:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983196800:12345-0@lo:400:kworker.0 Request processed in 50us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320942.534222:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016 00010000:00080000:0.0:1767320942.534335:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631908380 x1853163983196800/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320958 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320942.534349:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983196800:0@lo:400:kworker.0 00000100:00100000:1.0:1767320942.534444:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983196544:0@lo:400:kworker.0 00000100:00100000:1.0:1767320942.534478:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320942.534509:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320942.534514:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983197312:0@lo:400:kworker.0 00000100:00100000:1.0:1767320942.534532:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320942.534555:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320942.534634:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197184 00000100:00100000:3.0:1767320942.534639:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197184 00000100:00100000:3.0:1767320942.534679:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197312 00000100:00100000:3.0:1767320942.534682:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197312 00000100:00100000:3.0:1767320942.534699:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1233 00000100:00100000:3.0:1767320942.534705:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983197184:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.534749:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983197184:12345-0@lo:400:kworker.0 Request processed in 44us (696us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.534755:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1233 00000100:00100000:3.0:1767320942.534766:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1234 00000100:00100000:3.0:1767320942.534771:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983197312:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320942.534794:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983197312:12345-0@lo:400:kworker.0 Request processed in 22us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320942.534799:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1234 00000100:00100000:3.0:1767320942.534823:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983197184:0@lo:400:kworker.0 00000100:00080000:1.0:1767320942.534887:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320942.534890:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320942.534906:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983197312:0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.534975:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983196416:0@lo:250:kworker.0 00000400:00000100:2.0:1767320942.535012:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983196416 offset 0 length 520: 4 00000100:00100000:2.0:1767320942.535033:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983196672:0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.535054:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.535084:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983196928:0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.535101:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.535121:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983197056:0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.535143:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.535159:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631908e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983197440:0@lo:38:kworker.0 00000100:00100000:2.0:1767320942.535177:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320942.535205:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197056 00000100:00100000:2.0:1767320942.535208:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197056 00000100:00100000:2.0:1767320942.535229:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 692 00000100:00100000:2.0:1767320942.535235:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983197056:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.535256:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.535309:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983197056:12345-0@lo:38:kworker.0 Request processed in 73us (165us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.535315:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 692 00000100:00100000:2.0:1767320942.535332:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196544 00000100:00100000:2.0:1767320942.535335:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196544 00000100:00100000:2.0:1767320942.535357:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196672 00000100:00100000:2.0:1767320942.535359:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196672 00000100:00100000:2.0:1767320942.535366:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983196928 00000100:00100000:2.0:1767320942.535368:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983196928 00000100:00100000:2.0:1767320942.535374:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197440 00000100:00100000:2.0:1767320942.535376:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197440 00000100:00100000:2.0:1767320942.535384:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2017 00000100:00100000:2.0:1767320942.535389:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983196544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320942.535416:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983196544:12345-0@lo:400:kworker.0 Request processed in 27us (938us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320942.535421:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2017 00000100:00100000:2.0:1767320942.535430:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2669 00000100:00100000:2.0:1767320942.535433:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983196672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.535442:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.535460:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983196672:12345-0@lo:38:kworker.0 Request processed in 26us (406us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.535469:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2669 00000100:00100000:2.0:1767320942.535477:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2670 00000100:00100000:2.0:1767320942.535480:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983196928:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.535488:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.535503:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983196928:12345-0@lo:38:kworker.0 Request processed in 21us (401us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.535508:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2670 00000100:00100000:2.0:1767320942.535516:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2671 00000100:00100000:2.0:1767320942.535519:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983197440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320942.535527:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320942.535542:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983197440:12345-0@lo:38:kworker.0 Request processed in 21us (363us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320942.535547:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2671 00000100:00080000:2.0:1767320942.535576:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.535581:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.535584:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983196672:0@lo:38:kworker.0 00000100:00080000:2.0:1767320942.535591:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.535594:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.535597:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983196928:0@lo:38:kworker.0 00000100:00080000:2.0:1767320942.535603:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.535606:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.535609:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983197056:0@lo:38:kworker.0 00000100:00080000:2.0:1767320942.535614:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320942.535618:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320942.535620:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631908e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983197440:0@lo:38:kworker.0 00000100:00080000:1.0:1767320942.535689:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320942.535710:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463190b100 x1853163983196544/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320958 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320942.535721:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983196544:0@lo:400:kworker.0 00010000:00010000:2.0:1767320942.726444:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320943.750131:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320944.068549:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320944.068580:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983197568:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320944.068672:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320944.068754:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197568 00000100:00100000:3.0:1767320944.068772:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197568 00000100:00100000:3.0:1767320944.068871:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1346 00000100:00100000:3.0:1767320944.068890:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983197568:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767320944.068929:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767320944.069024:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983197568:12345-0@lo:13:osp-pre-0-1.0 Request processed in 144us (350us total) trans 0 rc 0/0 00000100:00080000:1.0:1767320944.069088:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320944.069132:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983197568:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320944.069185:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983197696:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320944.069225:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320944.069263:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197696 00000100:00100000:1.0:1767320944.069267:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197696 00000100:00080000:0.0:1767320944.069737:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320944.071054:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1346 00000100:00100000:1.0:1767320944.071347:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347 00000100:00100000:1.0:1767320944.071365:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983197696:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320944.071397:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320944.071478:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983197696:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (2251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320944.071485:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347 00000100:00100000:0.0:1767320944.071563:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983197696:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320944.526532:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320944.526641:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710080 00000100:00100000:3.0:1767320944.526657:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710080 00000100:00100000:3.0:1767320944.526816:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2018 00000100:00100000:3.0:1767320944.526829:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915710080:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320944.526915:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915710080:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 90us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320944.526921:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2018 00000100:00080000:0.0:1767320944.526974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320944.530561:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320944.530884:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915709952 00000100:00100000:3.0:1767320944.530896:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915709952 00000100:00100000:3.0:1767320944.530941:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2672 00000100:00100000:3.0:1767320944.531526:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915709952:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320944.531639:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320944.531807:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915709952:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 855us (1254us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320944.531816:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2672 00010000:00010000:3.0:1767320944.772708:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320945.796609:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320946.820883:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320947.653694:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320947.653709:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320947.653788:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320947.653793:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320947.653798:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320947.653810:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6191 00000100:00080000:2.0:1767320947.653816:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320947.653825:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320947.653854:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7480 x1853163983197952/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320947.653917:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320947.653921:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320947.653922:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320947.653926:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6191 00000100:00080000:2.0:1767320947.653928:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320947.653932:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320947.653934:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983197952:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.653961:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594a80 x1853163983198208/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320947.653970:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320947.653974:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320947.653976:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320947.653979:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6191 00000100:00080000:2.0:1767320947.653981:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320947.653984:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320947.653996:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163983198336/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320947.654050:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320947.654053:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320947.654055:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320947.654059:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6191 00000100:00080000:2.0:1767320947.654062:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320947.654065:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320947.654077:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631902d80 x1853163983198720/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320947.654093:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320947.654104:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983198464:0@lo:400:kworker.0 00000100:00100000:2.0:1767320947.654139:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.654139:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983198208:0@lo:38:kworker.0 00000100:00100000:3.0:1767320947.654161:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320947.654166:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320947.654172:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983198080:0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.654175:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983198336:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.654194:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.654196:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320947.654211:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983198720:0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.654225:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198464 00000100:00100000:3.0:1767320947.654233:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320947.654236:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198464 00000100:00100000:3.0:1767320947.654262:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197952 00000100:00100000:3.0:1767320947.654266:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197952 00000100:00100000:3.0:1767320947.654315:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198208 00000100:00100000:3.0:1767320947.654318:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198208 00000100:00100000:2.0:1767320947.654319:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1235 00000100:00100000:3.0:1767320947.654327:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198080 00000100:00100000:2.0:1767320947.654329:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983198464:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320947.654329:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198080 00000100:00100000:3.0:1767320947.654345:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198720 00000100:00100000:3.0:1767320947.654347:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198720 00000100:00100000:3.0:1767320947.654356:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2019 00000100:00100000:3.0:1767320947.654360:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983198080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320947.654385:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983198464:12345-0@lo:400:kworker.0 Request processed in 57us (244us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320947.654385:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983198080:12345-0@lo:400:kworker.0 Request processed in 24us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320947.654394:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1235 00000100:00100000:3.0:1767320947.654394:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2019 00000100:00100000:3.0:1767320947.654412:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2673 00000100:00100000:3.0:1767320947.654416:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983197952:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767320947.654419:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198336 00000100:00100000:2.0:1767320947.654422:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198336 00000100:00100000:2.0:1767320947.654440:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 693 00000100:00100000:2.0:1767320947.654444:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983198336:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320947.654503:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983197824:0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.654535:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320947.654555:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320947.654560:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983198592:0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.654578:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320947.654631:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983197824 00000100:00100000:0.0:1767320947.654635:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983197824 00000100:00100000:0.0:1767320947.654658:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2020 00000100:00100000:0.0:1767320947.654664:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983197824:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320947.654698:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983197824:12345-0@lo:400:kworker.0 Request processed in 34us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320947.654705:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2020 00010000:00020000:3.0:1767320947.654711:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320947.654717:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2674 00000100:00100000:0.0:1767320947.654721:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983198208:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320947.654745:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983197952:12345-0@lo:38:kworker.0 Request processed in 327us (652us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320947.654753:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2673 00000100:00100000:3.0:1767320947.654764:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2675 00000100:00100000:3.0:1767320947.654768:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983198720:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767320947.654791:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198592 00000100:00100000:1.0:1767320947.654795:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198592 00000100:00100000:1.0:1767320947.654826:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1236 00000100:00100000:1.0:1767320947.654833:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983198592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320947.654868:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983198592:12345-0@lo:400:kworker.0 Request processed in 37us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320947.654874:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1236 00000100:00080000:1.0:1767320947.654936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320947.654941:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767320947.654945:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767320947.654983:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983198592:0@lo:400:kworker.0 00010000:00080000:1.0:1767320947.654995:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4e00 x1853163983197824/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320963 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320947.655004:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983197824:0@lo:400:kworker.0 00010000:00080000:1.0:1767320947.655046:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6a00 x1853163983198080/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320963 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320947.655055:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983198080:0@lo:400:kworker.0 00000100:00100000:1.0:1767320947.655067:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983198464:0@lo:400:kworker.0 00010000:00020000:2.0:1767320947.655210:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320947.655249:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983198336:12345-0@lo:38:kworker.0 Request processed in 803us (1050us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320947.655256:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 693 00000100:00080000:2.0:1767320947.655332:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.655337:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.655340:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983198336:0@lo:38:kworker.0 00000100:00080000:2.0:1767320947.655349:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.655353:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.655355:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983197952:0@lo:38:kworker.0 00010000:00020000:0.0:1767320947.655806:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320947.655855:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983198208:12345-0@lo:38:kworker.0 Request processed in 1132us (1693us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320947.655864:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2674 00000100:00080000:2.0:1767320947.656060:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.656065:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.656069:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983198208:0@lo:38:kworker.0 00010000:00020000:3.0:1767320947.658196:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320947.658431:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ebb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983198720:12345-0@lo:38:kworker.0 Request processed in 3656us (4191us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320947.658447:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2675 00000100:00080000:2.0:1767320947.658930:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320947.658937:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320947.659164:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983198720:0@lo:38:kworker.0 00010000:00010000:3.0:1767320947.845062:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320948.869456:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320949.190743:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320949.190778:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983198848:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320949.190891:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320949.191061:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198848 00000100:00100000:1.0:1767320949.191072:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198848 00000100:00100000:1.0:1767320949.191152:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1348 00000100:00100000:1.0:1767320949.191163:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983198848:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320949.191206:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320949.191295:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983198848:12345-0@lo:13:osp-pre-0-1.0 Request processed in 110us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320949.191305:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1348 00000100:00100000:3.0:1767320949.191470:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983198848:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320949.191563:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320949.194092:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320949.194121:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983198976:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320949.194208:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320949.194376:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983198976 00000100:00100000:0.0:1767320949.194386:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983198976 00000100:00100000:0.0:1767320949.194486:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1349 00000100:00100000:0.0:1767320949.194497:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983198976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320949.194532:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320949.194593:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983198976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320949.194601:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1349 00000100:00100000:0.0:1767320949.194656:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983198976:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320949.194742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320949.643850:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320949.643979:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710336 00000100:00100000:3.0:1767320949.644149:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710336 00000100:00100000:3.0:1767320949.644248:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2021 00000100:00100000:3.0:1767320949.644261:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915710336:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320949.644408:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915710336:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 150us (577us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320949.644416:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2021 00000100:00080000:2.0:1767320949.644707:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320949.653655:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320949.654210:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710208 00000100:00100000:0.0:1767320949.654225:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710208 00000100:00100000:0.0:1767320949.654345:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2676 00000100:00100000:0.0:1767320949.654356:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915710208:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320949.654387:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320949.654481:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915710208:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 126us (841us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320949.654490:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2676 00010000:00010000:3.0:1767320949.892497:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320950.917250:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320951.942629:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320952.773631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320952.773646:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320952.773724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320952.773729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320952.773738:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320952.773751:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6196 00000100:00080000:1.0:1767320952.773757:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320952.773766:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320952.773810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d880 x1853163983199232/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320952.773838:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767320952.773861:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983199104:0@lo:400:kworker.0 00000100:00080000:1.0:1767320952.773867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320952.773870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320952.773872:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320952.773875:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6196 00000100:00080000:1.0:1767320952.773879:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320952.773892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320952.773906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469c000 x1853163983199488/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320952.773912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320952.773917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320952.773918:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320952.773921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6196 00000100:00080000:1.0:1767320952.773923:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320952.773926:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320952.773939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469f800 x1853163983199616/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320952.773958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320952.773961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320952.773964:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767320952.773967:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767320952.773973:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6196 00000100:00080000:1.0:1767320952.773976:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320952.773979:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320952.773995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee6300 x1853163983200000/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320952.774018:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983199872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320952.774032:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983199360:0@lo:400:kworker.0 00000100:00100000:2.0:1767320952.774050:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320952.774075:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320952.774116:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199872 00000100:00100000:2.0:1767320952.774126:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199872 00000100:00100000:1.0:1767320952.774180:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320952.774185:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983199744:0@lo:400:kworker.0 00000100:00100000:2.0:1767320952.774202:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1237 00000100:00100000:1.0:1767320952.774212:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320952.774218:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983199872:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320952.774244:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199744 00000100:00100000:1.0:1767320952.774248:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199744 00000100:00100000:1.0:1767320952.774311:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1238 00000100:00100000:2.0:1767320952.774316:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983199872:12345-0@lo:400:kworker.0 Request processed in 102us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320952.774324:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983199744:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320952.774329:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1237 00000100:00080000:2.0:1767320952.774360:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320952.774365:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320952.774377:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983199744:12345-0@lo:400:kworker.0 Request processed in 58us (165us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320952.774384:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1238 00000100:00100000:2.0:1767320952.774403:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983199872:0@lo:400:kworker.0 00000100:00100000:1.0:1767320952.774432:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983199744:0@lo:400:kworker.0 00000100:00100000:0.0:1767320952.774525:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199104 00000100:00100000:0.0:1767320952.774529:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199104 00000100:00100000:0.0:1767320952.774585:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199360 00000100:00100000:0.0:1767320952.774592:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199360 00000100:00100000:0.0:1767320952.774612:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022 00000100:00100000:0.0:1767320952.774618:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983199104:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320952.774658:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983199104:12345-0@lo:400:kworker.0 Request processed in 41us (696us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320952.774663:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022 00000100:00100000:0.0:1767320952.774673:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2023 00000100:00100000:0.0:1767320952.774677:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983199360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320952.774697:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983199360:12345-0@lo:400:kworker.0 Request processed in 20us (623us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320952.774702:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2023 00010000:00080000:0.0:1767320952.774748:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469ce00 x1853163983199104/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320968 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320952.774762:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983199104:0@lo:400:kworker.0 00000100:00100000:3.0:1767320952.774843:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983199232:0@lo:38:kworker.0 00000100:00100000:3.0:1767320952.774875:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320952.774907:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983199488:0@lo:38:kworker.0 00000100:00100000:3.0:1767320952.774924:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320952.774937:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983199616:0@lo:38:kworker.0 00000100:00100000:3.0:1767320952.774963:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320952.774999:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983200000:0@lo:38:kworker.0 00000100:00100000:3.0:1767320952.775022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320952.775048:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199232 00000100:00100000:3.0:1767320952.775052:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199232 00000100:00100000:3.0:1767320952.775070:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199488 00000100:00100000:3.0:1767320952.775073:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199488 00000100:00100000:3.0:1767320952.775079:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200000 00000100:00100000:3.0:1767320952.775081:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200000 00000100:00100000:3.0:1767320952.775088:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2677 00000100:00100000:3.0:1767320952.775092:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983199232:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320952.775112:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320952.775143:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983199232:12345-0@lo:38:kworker.0 Request processed in 50us (267us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320952.775149:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2677 00000100:00100000:3.0:1767320952.775160:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2678 00000100:00100000:3.0:1767320952.775162:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983199488:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320952.775169:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320952.775184:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983199488:12345-0@lo:38:kworker.0 Request processed in 21us (259us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320952.775188:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2678 00000100:00100000:3.0:1767320952.775195:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2679 00000100:00100000:3.0:1767320952.775198:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983200000:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320952.775205:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320952.775218:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983200000:12345-0@lo:38:kworker.0 Request processed in 19us (196us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320952.775223:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2679 00000100:00080000:3.0:1767320952.775242:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320952.775247:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320952.775250:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983199232:0@lo:38:kworker.0 00000100:00080000:3.0:1767320952.775256:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320952.775258:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320952.775261:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983199488:0@lo:38:kworker.0 00000100:00080000:3.0:1767320952.775266:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320952.775288:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320952.775290:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983200000:0@lo:38:kworker.0 00010000:00080000:1.0:1767320952.775783:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469c380 x1853163983199360/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320968 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320952.775794:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983199360:0@lo:400:kworker.0 00000100:00100000:2.0:1767320952.776478:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983199616 00000100:00100000:2.0:1767320952.776488:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983199616 00000100:00100000:2.0:1767320952.776534:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 694 00000100:00100000:2.0:1767320952.776543:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983199616:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320952.776563:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320952.776646:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983199616:12345-0@lo:38:kworker.0 Request processed in 104us (1681us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320952.776654:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 694 00000100:00080000:2.0:1767320952.776677:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320952.776868:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320952.776874:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320952.776878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983199616:0@lo:38:kworker.0 00010000:00010000:3.0:1767320952.964729:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320953.989176:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320954.309333:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983200128:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320954.309421:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320954.309543:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200128 00000100:00100000:1.0:1767320954.309553:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200128 00000100:00100000:1.0:1767320954.309619:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1350 00000100:00100000:1.0:1767320954.309629:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983200128:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320954.309672:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320954.309720:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983200128:12345-0@lo:13:osp-pre-0-1.0 Request processed in 93us (299us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320954.309726:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1350 00000100:00080000:0.0:1767320954.309827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767320954.309874:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983200128:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320954.310323:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983200256:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320954.310353:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320954.310506:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200256 00000100:00100000:2.0:1767320954.310512:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200256 00000100:00100000:2.0:1767320954.310550:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1351 00000100:00100000:2.0:1767320954.310555:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983200256:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320954.310574:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320954.310614:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983200256:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320954.310619:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1351 00000100:00100000:3.0:1767320954.310679:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983200256:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767320954.310739:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320954.764460:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320954.764643:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320954.765484:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710592 00000100:00100000:2.0:1767320954.765500:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710592 00000100:00100000:2.0:1767320954.765644:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710464 00000100:00100000:2.0:1767320954.765657:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710464 00000100:00100000:2.0:1767320954.765698:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2024 00000100:00100000:2.0:1767320954.765712:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915710592:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320954.765852:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915710592:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 143us (1410us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320954.765861:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2024 00000100:00100000:2.0:1767320954.765887:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2680 00000100:00100000:2.0:1767320954.765893:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915710464:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767320954.765920:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320954.765955:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915710464:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 60us (1312us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320954.765962:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2680 00000100:00080000:0.0:1767320954.766816:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320955.012883:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320956.036728:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320957.061208:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320957.892571:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320957.892587:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320957.892680:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320957.892684:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320957.892689:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320957.892699:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6201 00000100:00080000:2.0:1767320957.892714:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320957.892724:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320957.892746:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320957.892756:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900380 x1853163983200512/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320957.892768:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983200384:0@lo:400:kworker.0 00000100:00080000:2.0:1767320957.892811:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320957.892816:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320957.892818:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320957.892822:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6201 00000100:00080000:2.0:1767320957.892825:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320957.892830:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320957.892842:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903800 x1853163983200768/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320957.892852:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320957.892856:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320957.892857:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320957.892860:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6201 00000100:00100000:1.0:1767320957.892863:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983200512:0@lo:38:kworker.0 00000100:00080000:2.0:1767320957.892863:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320957.892867:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320957.892881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320957.892883:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901c00 x1853163983200896/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320957.892924:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767320957.892936:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983200640:0@lo:400:kworker.0 00000100:00100000:0.0:1767320957.892978:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200384 00000100:00100000:0.0:1767320957.893002:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200384 00000100:00100000:0.0:1767320957.893076:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025 00000100:00100000:0.0:1767320957.893089:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983200384:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767320957.893580:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320957.893585:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320957.893587:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320957.893592:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6201 00000100:00080000:2.0:1767320957.893595:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320957.893600:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320957.893615:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163983201280/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320957.893882:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320957.893894:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983201152:0@lo:400:kworker.0 00000100:00100000:1.0:1767320957.896935:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.897053:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983201024:0@lo:400:kworker.0 00000100:00100000:1.0:1767320957.897098:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.897149:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201024 00000100:00100000:1.0:1767320957.898367:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201024 00000100:00100000:1.0:1767320957.898471:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1239 00000100:00100000:1.0:1767320957.898483:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983201024:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320957.898596:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983201024:12345-0@lo:400:kworker.0 Request processed in 114us (1496us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320957.898605:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1239 00000100:00100000:1.0:1767320957.898653:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200640 00000100:00100000:1.0:1767320957.898659:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200640 00000100:00100000:1.0:1767320957.898698:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2026 00000100:00100000:1.0:1767320957.898731:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983200640:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320957.898767:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983200640:12345-0@lo:400:kworker.0 Request processed in 38us (1844us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320957.898774:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2026 00000100:00100000:1.0:1767320957.898806:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.898853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983200768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320957.898894:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.898920:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983200896:0@lo:38:kworker.0 00000100:00100000:1.0:1767320957.898948:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.898975:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983201280:0@lo:38:kworker.0 00000100:00100000:1.0:1767320957.898999:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.899031:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200896 00000100:00100000:1.0:1767320957.899036:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200896 00000100:00100000:1.0:1767320957.899059:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 695 00000100:00100000:1.0:1767320957.899063:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983200896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320957.899087:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320957.899120:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983200896:12345-0@lo:38:kworker.0 Request processed in 55us (171us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320957.899126:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 695 00000100:00100000:1.0:1767320957.899143:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200512 00000100:00100000:1.0:1767320957.899147:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200512 00000100:00100000:1.0:1767320957.902344:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983200768 00000100:00100000:1.0:1767320957.902369:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983200768 00000100:00100000:1.0:1767320957.902410:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201280 00000100:00100000:1.0:1767320957.902414:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201280 00000100:00100000:1.0:1767320957.902435:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2681 00000100:00100000:1.0:1767320957.902445:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983200512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320957.902478:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320957.902594:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e245880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983200512:12345-0@lo:38:kworker.0 Request processed in 149us (3785us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320957.902602:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2681 00000100:00100000:1.0:1767320957.902623:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2682 00000100:00100000:1.0:1767320957.902627:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983200768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320957.902637:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320957.902666:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983200768:12345-0@lo:38:kworker.0 Request processed in 37us (3771us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320957.902672:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2682 00000100:00100000:1.0:1767320957.902682:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2683 00000100:00100000:1.0:1767320957.902686:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983201280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320957.902696:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320957.902732:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983201280:12345-0@lo:38:kworker.0 Request processed in 44us (3732us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320957.902739:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2683 00000100:00080000:1.0:1767320957.902834:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320957.902841:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320957.902847:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983200896:0@lo:38:kworker.0 00000100:00080000:1.0:1767320957.902860:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320957.902864:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320957.902867:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983201280:0@lo:38:kworker.0 00000100:00080000:1.0:1767320957.902879:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320957.902882:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320957.902885:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983200512:0@lo:38:kworker.0 00000100:00080000:1.0:1767320957.902895:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320957.902898:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320957.902901:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983200768:0@lo:38:kworker.0 00000100:00100000:1.0:1767320957.902964:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983201024:0@lo:400:kworker.0 00010000:00080000:1.0:1767320957.902983:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900000 x1853163983200640/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320973 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320957.902996:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983200640:0@lo:400:kworker.0 00000100:00080000:1.0:1767320957.903051:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767320957.903057:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320957.905309:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983200384:12345-0@lo:400:kworker.0 Request processed in 12183us (12389us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320957.905334:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025 00010000:00080000:3.0:1767320957.907428:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901f80 x1853163983200384/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320973 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320957.907455:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983200384:0@lo:400:kworker.0 00000100:00100000:2.0:1767320957.910317:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320957.910728:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201152 00000100:00100000:1.0:1767320957.910741:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201152 00000100:00100000:1.0:1767320957.910848:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1240 00000100:00100000:1.0:1767320957.910860:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983201152:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320957.910957:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983201152:12345-0@lo:400:kworker.0 Request processed in 100us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320957.910967:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1240 00000100:00100000:3.0:1767320957.911030:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983201152:0@lo:400:kworker.0 00000100:00080000:0.0:1767320957.911116:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320958.084490:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320959.108692:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320959.428730:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767320959.428767:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983201408:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320959.428857:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320959.429241:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201408 00000100:00100000:1.0:1767320959.429250:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201408 00000100:00100000:1.0:1767320959.429336:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352 00000100:00100000:1.0:1767320959.429345:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983201408:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320959.429388:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320959.429431:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983201408:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (575us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320959.429439:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352 00000100:00100000:0.0:1767320959.429532:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983201408:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320959.432996:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320959.433429:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320959.433453:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983201536:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320959.433529:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320959.433680:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201536 00000100:00100000:2.0:1767320959.433689:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201536 00000100:00100000:2.0:1767320959.433755:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1353 00000100:00100000:2.0:1767320959.433765:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983201536:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767320959.433994:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:2.0:1767320959.434167:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320959.434231:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983201536:12345-0@lo:13:osp-pre-1-1.0 Request processed in 467us (702us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320959.434238:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1353 00000100:00100000:3.0:1767320959.434337:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983201536:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320959.885624:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320959.885721:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320959.885815:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710848 00000100:00100000:1.0:1767320959.885826:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710848 00000100:00100000:1.0:1767320959.885895:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915710720 00000100:00100000:1.0:1767320959.885899:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915710720 00000100:00100000:1.0:1767320959.885919:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2027 00000100:00100000:1.0:1767320959.885929:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915710848:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320959.886016:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915710848:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320959.886024:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2027 00000100:00100000:1.0:1767320959.886043:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2684 00000100:00100000:1.0:1767320959.886047:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915710720:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320959.886068:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320959.886095:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915710720:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (373us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320959.886101:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2684 00000100:00080000:2.0:1767320959.886242:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320960.132552:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320961.156579:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320962.184831:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320963.013225:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320963.013239:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320963.013356:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320963.013360:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320963.013364:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320963.013372:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6206 00000100:00080000:2.0:1767320963.013377:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320963.013383:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320963.013405:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903800 x1853163983201792/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320963.013435:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320963.013438:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320963.013440:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320963.013443:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6206 00000100:00080000:2.0:1767320963.013446:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320963.013449:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320963.013460:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163983202048/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320963.013468:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320963.013472:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320963.013473:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320963.013476:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6206 00000100:00080000:2.0:1767320963.013478:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320963.013482:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320963.013490:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901c00 x1853163983202176/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320963.013508:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320963.013511:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320963.013513:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320963.013516:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6206 00000100:00080000:2.0:1767320963.013518:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320963.013521:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320963.013528:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901180 x1853163983202560/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320963.013574:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983202304:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.013654:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320963.013708:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320963.013714:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983201920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.013745:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320963.013791:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202304 00000100:00100000:2.0:1767320963.013799:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202304 00000100:00100000:2.0:1767320963.013873:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1241 00000100:00100000:2.0:1767320963.013881:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983202304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.013928:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983202304:12345-0@lo:400:kworker.0 Request processed in 48us (273us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320963.013936:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1241 00000100:00100000:2.0:1767320963.013968:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201920 00000100:00100000:2.0:1767320963.013971:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201920 00000100:00100000:2.0:1767320963.013999:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028 00000100:00100000:2.0:1767320963.014004:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983201920:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.014029:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983201920:12345-0@lo:400:kworker.0 Request processed in 25us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320963.014035:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028 00000100:00080000:2.0:1767320963.014050:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320963.014054:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:2.0:1767320963.014076:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900380 x1853163983201920/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320979 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320963.014090:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983201920:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.014098:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983202304:0@lo:400:kworker.0 00000100:00100000:1.0:1767320963.014240:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983201792:0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.014288:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.014317:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983202048:0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.014334:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.014347:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983202176:0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.014368:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.014387:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983202560:0@lo:38:kworker.0 00000100:00100000:1.0:1767320963.014404:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320963.014442:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202176 00000100:00100000:1.0:1767320963.014446:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202176 00000100:00100000:1.0:1767320963.014469:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 696 00000100:00100000:1.0:1767320963.014474:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983202176:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320963.014493:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.014523:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983202176:12345-0@lo:38:kworker.0 Request processed in 49us (155us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.014529:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 696 00000100:00100000:1.0:1767320963.014548:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201792 00000100:00100000:1.0:1767320963.014550:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201792 00000100:00100000:1.0:1767320963.014563:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202048 00000100:00100000:1.0:1767320963.014565:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202048 00000100:00100000:1.0:1767320963.014574:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202560 00000100:00100000:1.0:1767320963.014576:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202560 00000100:00100000:1.0:1767320963.014584:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2685 00000100:00100000:1.0:1767320963.014589:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983201792:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320963.014597:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.014631:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983201792:12345-0@lo:38:kworker.0 Request processed in 41us (361us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.014637:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2685 00000100:00100000:1.0:1767320963.014646:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2686 00000100:00100000:1.0:1767320963.014650:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983202048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320963.014657:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.014674:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983202048:12345-0@lo:38:kworker.0 Request processed in 23us (339us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.014679:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2686 00000100:00100000:1.0:1767320963.014687:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2687 00000100:00100000:1.0:1767320963.014690:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983202560:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320963.014698:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.014711:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983202560:12345-0@lo:38:kworker.0 Request processed in 20us (307us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.014716:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2687 00000100:00080000:1.0:1767320963.014741:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320963.014746:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320963.014749:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983201792:0@lo:38:kworker.0 00000100:00080000:1.0:1767320963.014769:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320963.014773:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320963.014776:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983202048:0@lo:38:kworker.0 00000100:00080000:1.0:1767320963.014783:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320963.014797:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320963.014800:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983202176:0@lo:38:kworker.0 00000100:00080000:1.0:1767320963.014807:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767320963.014810:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767320963.014813:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983202560:0@lo:38:kworker.0 00000100:00100000:3.0:1767320963.015078:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983201664:0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.015111:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320963.015139:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767320963.015145:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983202432:0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.015163:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320963.015199:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202432 00000100:00100000:3.0:1767320963.015204:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202432 00000100:00100000:3.0:1767320963.015236:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1242 00000100:00100000:3.0:1767320963.015242:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983202432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320963.015290:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983202432:12345-0@lo:400:kworker.0 Request processed in 49us (126us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320963.015296:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1242 00000100:00080000:3.0:1767320963.015311:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320963.015327:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983202432:0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.015546:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983201664 00000100:00100000:2.0:1767320963.015550:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983201664 00000100:00100000:2.0:1767320963.015586:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2029 00000100:00100000:2.0:1767320963.015592:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983201664:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320963.015638:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983201664:12345-0@lo:400:kworker.0 Request processed in 46us (525us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320963.015644:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2029 00000100:00080000:3.0:1767320963.018173:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320963.018250:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631902680 x1853163983201664/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320979 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320963.018317:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631902680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983201664:0@lo:400:kworker.0 00010000:00010000:3.0:1767320963.204596:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1767320963.983471:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915710976 offset 0 length 520: 4 00000100:00100000:2.0:1767320963.991711:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320963.991770:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320963.991897:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711104 00000100:00100000:1.0:1767320963.991909:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711104 00000100:00100000:1.0:1767320963.991954:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711232 00000100:00100000:1.0:1767320963.991957:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711232 00000100:00100000:1.0:1767320963.992011:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2030 00000100:00100000:1.0:1767320963.992021:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915711232:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320963.992133:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915711232:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 113us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320963.992142:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2030 00000100:00100000:1.0:1767320963.992164:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2688 00000100:00100000:1.0:1767320963.992168:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915711104:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767320963.992193:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320963.992223:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915711104:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (528us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320963.992229:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2688 00000100:00080000:0.0:1767320963.993048:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320964.228574:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320964.549488:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983202688:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767320964.549573:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320964.550061:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202688 00000100:00100000:1.0:1767320964.550070:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202688 00000100:00100000:1.0:1767320964.550118:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1354 00000100:00100000:1.0:1767320964.550127:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983202688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320964.550158:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320964.550214:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983202688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (641us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320964.550221:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1354 00000100:00100000:1.0:1767320964.550293:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983202688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320964.550340:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983202816:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320964.550372:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320964.551015:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983202816 00000100:00100000:2.0:1767320964.551021:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983202816 00000100:00100000:2.0:1767320964.551059:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1355 00000100:00100000:2.0:1767320964.551067:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983202816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320964.551086:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320964.551133:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983202816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (760us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320964.551144:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1355 00000100:00100000:2.0:1767320964.552799:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983202816:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320965.252831:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320966.277324:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320967.110702:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320942/real 1767320942] req@ffff93463190aa00 x1853163983196416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320967 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320967.110737:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463190aa00 x1853163983196416/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320967 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767320967.110758:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320967.110785:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767320967.110789:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983196416:0@lo:250:kworker.0 00000100:00080000:3.0:1767320967.110885:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320967.110891:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767320967.110894:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320967.110899:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6191 00000100:00080000:3.0:1767320967.110903:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320967.110909:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320967.110941:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963100 x1853163983202944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320967.110990:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320967.110994:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320967.110996:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320967.111000:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6211 00000100:00080000:3.0:1767320967.111002:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320967.111006:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320967.111020:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960a80 x1853163983203200/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320967.111047:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320967.111050:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320967.111052:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320967.111056:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6211 00000100:00080000:3.0:1767320967.111059:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320967.111062:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320967.111073:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f961500 x1853163983203456/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320967.111081:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320967.111084:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320967.111086:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320967.111088:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6211 00000100:00080000:3.0:1767320967.111091:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320967.111094:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320967.111103:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f960380 x1853163983203584/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320967.111121:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320967.111124:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320967.111126:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320967.111130:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6211 00000100:00080000:3.0:1767320967.111132:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320967.111135:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320967.111142:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463f963480 x1853163983203968/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320967.111167:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983203328:0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.111231:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320967.111309:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320967.111315:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983203712:0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.111342:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320967.111375:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203328 00000100:00100000:3.0:1767320967.111382:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203328 00000100:00100000:3.0:1767320967.111434:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031 00000100:00100000:3.0:1767320967.111442:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983203328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.111496:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983203328:12345-0@lo:400:kworker.0 Request processed in 55us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320967.111504:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031 00000100:00100000:3.0:1767320967.111527:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203712 00000100:00100000:3.0:1767320967.111542:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203712 00000100:00100000:3.0:1767320967.111572:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1243 00000100:00100000:3.0:1767320967.111578:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983203712:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320967.111610:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983203712:12345-0@lo:400:kworker.0 Request processed in 32us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320967.111616:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1243 00000100:00100000:3.0:1767320967.111658:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983203712:0@lo:400:kworker.0 00010000:00080000:3.0:1767320967.111669:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f962680 x1853163983203328/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320983 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320967.111677:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983203328:0@lo:400:kworker.0 00000100:00100000:2.0:1767320967.111772:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983202944:0@lo:250:kworker.0 00000400:00000100:2.0:1767320967.111795:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983202944 offset 0 length 520: 4 00000100:00100000:2.0:1767320967.111816:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983203200:0@lo:38:kworker.0 00000100:00100000:2.0:1767320967.111836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320967.111863:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983203456:0@lo:38:kworker.0 00000100:00100000:2.0:1767320967.111881:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320967.111897:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983203584:0@lo:38:kworker.0 00000100:00100000:2.0:1767320967.111918:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320967.111934:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983203968:0@lo:38:kworker.0 00000100:00100000:2.0:1767320967.111953:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320967.111977:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203200 00000100:00100000:2.0:1767320967.111981:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203200 00000100:00100000:2.0:1767320967.111997:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203456 00000100:00100000:2.0:1767320967.111999:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203456 00000100:00100000:2.0:1767320967.112010:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2689 00000100:00100000:2.0:1767320967.112014:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320967.112030:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320967.112061:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203200:12345-0@lo:38:kworker.0 Request processed in 45us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320967.112068:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2689 00000100:00100000:2.0:1767320967.112077:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203968 00000100:00100000:2.0:1767320967.112080:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203968 00000100:00100000:2.0:1767320967.112090:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2690 00000100:00100000:2.0:1767320967.112093:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203456:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320967.112102:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320967.112124:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203456:12345-0@lo:38:kworker.0 Request processed in 29us (241us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320967.112129:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2690 00000100:00100000:2.0:1767320967.112140:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2691 00000100:00100000:2.0:1767320967.112143:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203968:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320967.112151:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320967.112173:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983203968:12345-0@lo:38:kworker.0 Request processed in 28us (220us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320967.112179:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2691 00000100:00080000:2.0:1767320967.112200:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320967.112204:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320967.112208:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983203200:0@lo:38:kworker.0 00000100:00080000:2.0:1767320967.112215:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320967.112218:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320967.112221:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f961500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983203456:0@lo:38:kworker.0 00000100:00080000:2.0:1767320967.112228:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320967.112231:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320967.112233:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983203968:0@lo:38:kworker.0 00000100:00100000:0.0:1767320967.112334:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983203072:0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.112363:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320967.112382:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320967.112386:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983203840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.112405:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767320967.112425:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320967.112429:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767320967.112437:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203840 00000100:00100000:0.0:1767320967.112440:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203840 00000100:00100000:0.0:1767320967.112468:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1244 00000100:00100000:0.0:1767320967.112474:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983203840:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.112505:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983203840:12345-0@lo:400:kworker.0 Request processed in 32us (100us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.112511:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1244 00000100:00080000:0.0:1767320967.112523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320967.112555:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983203840:0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.112570:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203072 00000100:00100000:0.0:1767320967.112573:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203072 00000100:00100000:0.0:1767320967.112593:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2032 00000100:00100000:0.0:1767320967.112599:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983203072:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320967.112630:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983203072:12345-0@lo:400:kworker.0 Request processed in 32us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320967.112637:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2032 00000100:00080000:0.0:1767320967.112649:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767320967.112664:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463f960000 x1853163983203072/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320983 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320967.112674:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983203072:0@lo:400:kworker.0 00000100:00100000:1.0:1767320967.115179:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983203584 00000100:00100000:1.0:1767320967.115196:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983203584 00000100:00100000:1.0:1767320967.115255:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 697 00000100:00100000:1.0:1767320967.115264:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983203584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320967.115334:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320967.115431:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983203584:12345-0@lo:38:kworker.0 Request processed in 167us (3511us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320967.115437:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 697 00000100:00080000:3.0:1767320967.115779:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320967.115797:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320967.115801:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983203584:0@lo:38:kworker.0 00010000:00010000:2.0:1767320967.300774:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320968.325649:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320969.117762:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320969.117871:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320969.121163:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711488 00000100:00100000:3.0:1767320969.121179:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711488 00000100:00100000:1.0:1767320969.121421:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711360 00000100:00100000:1.0:1767320969.121426:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711360 00000100:00100000:1.0:1767320969.121657:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2033 00000100:00100000:1.0:1767320969.121670:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915711488:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767320969.121755:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915711488:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (4003us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320969.121763:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2033 00000100:00100000:1.0:1767320969.121783:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2692 00000100:00100000:1.0:1767320969.121787:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915711360:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:0.0:1767320969.121807:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:1.0:1767320969.121808:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320969.121837:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915711360:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (3964us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320969.121844:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2692 00010000:00010000:3.0:1767320969.348473:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320969.668796:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983204096:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320969.668876:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320969.669307:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204096 00000100:00100000:2.0:1767320969.669316:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204096 00000100:00100000:2.0:1767320969.669388:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1356 00000100:00100000:2.0:1767320969.669398:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983204096:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320969.669431:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320969.669480:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983204096:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (604us total) trans 0 rc 0/0 00000100:00080000:0.0:1767320969.669690:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320969.669730:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983204096:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767320969.669783:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1356 00000100:00100000:3.0:1767320969.677316:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983204224:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320969.677387:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320969.677549:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204224 00000100:00100000:2.0:1767320969.677557:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204224 00000100:00100000:2.0:1767320969.677620:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1357 00000100:00100000:2.0:1767320969.677630:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983204224:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320969.677662:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320969.677719:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983204224:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (331us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320969.677727:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1357 00000100:00100000:1.0:1767320969.677813:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983204224:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767320969.677922:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767320970.375751:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320971.399792:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320972.230342:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320972.230354:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320972.230477:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320972.230492:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320972.230496:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320972.230504:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6215 00000100:00080000:2.0:1767320972.230509:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320972.230525:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320972.230549:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163983204480/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320972.230609:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320972.230612:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320972.230614:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320972.230618:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6215 00000100:00080000:2.0:1767320972.230620:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320972.230625:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320972.230635:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631903100 x1853163983204736/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320972.230645:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320972.230648:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320972.230650:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320972.230653:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6215 00000100:00080000:2.0:1767320972.230655:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320972.230658:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320972.230689:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900380 x1853163983204864/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320972.230724:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320972.230727:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320972.230729:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320972.230732:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6215 00000100:00080000:2.0:1767320972.230735:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320972.230738:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767320972.230896:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983204480:0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.230994:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231050:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983204736:0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.231071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231091:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983204864:0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.231126:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231175:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983204992:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.231203:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231223:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320972.231228:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983204608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.231249:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231321:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204480 00000100:00100000:0.0:1767320972.231328:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204480 00000100:00100000:0.0:1767320972.231360:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204736 00000100:00100000:0.0:1767320972.231363:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204736 00000100:00100000:0.0:1767320972.231376:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204608 00000100:00100000:0.0:1767320972.231379:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204608 00000100:00100000:0.0:1767320972.231411:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034 00000100:00100000:0.0:1767320972.231420:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983204608:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.231484:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2693 00000100:00100000:0.0:1767320972.231493:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983204608:12345-0@lo:400:kworker.0 Request processed in 72us (240us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320972.231497:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983204480:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.231505:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034 00000100:00100000:0.0:1767320972.231525:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2694 00000100:00100000:0.0:1767320972.231529:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983204736:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320972.231537:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:0.0:1767320972.231541:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320972.231588:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983204736:12345-0@lo:38:kworker.0 Request processed in 56us (515us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320972.231593:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983204480:12345-0@lo:38:kworker.0 Request processed in 97us (597us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320972.231599:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2694 00000100:00100000:3.0:1767320972.231607:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2693 00000100:00100000:0.0:1767320972.231623:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204864 00000100:00100000:0.0:1767320972.231628:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204864 00000100:00100000:0.0:1767320972.231647:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 698 00000100:00100000:3.0:1767320972.231648:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983205120:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.231654:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983204864:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320972.231665:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320972.231683:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320972.231697:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983204864:12345-0@lo:38:kworker.0 Request processed in 43us (569us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320972.231704:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 698 00000100:00100000:0.0:1767320972.231724:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204992 00000100:00100000:0.0:1767320972.231728:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204992 00000100:00100000:3.0:1767320972.231728:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320972.231738:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983204352:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.231756:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205120 00000100:00100000:0.0:1767320972.231760:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205120 00000100:00100000:3.0:1767320972.231765:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320972.231794:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983204352 00000100:00100000:3.0:1767320972.231798:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983204352 00000100:00100000:0.0:1767320972.231811:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1245 00000100:00100000:0.0:1767320972.231820:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983204992:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.231832:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2035 00000100:00100000:3.0:1767320972.231842:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983204352:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.231853:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983204992:12345-0@lo:400:kworker.0 Request processed in 34us (648us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320972.231860:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1245 00000100:00100000:0.0:1767320972.231872:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1246 00000100:00100000:0.0:1767320972.231879:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983205120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.231887:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983204352:12345-0@lo:400:kworker.0 Request processed in 45us (121us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320972.231894:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2035 00000100:00100000:0.0:1767320972.231909:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983205120:12345-0@lo:400:kworker.0 Request processed in 30us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320972.231914:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1246 00000100:00080000:0.0:1767320972.231942:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767320972.231946:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767320972.231949:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767320972.231953:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631901c00 x1853163983204352/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320988 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320972.231968:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983204352:0@lo:400:kworker.0 00000100:00100000:3.0:1767320972.231978:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983205120:0@lo:400:kworker.0 00010000:00080000:0.0:1767320972.232019:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631903800 x1853163983204608/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320988 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320972.232031:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983204608:0@lo:400:kworker.0 00000100:00100000:0.0:1767320972.232043:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983204992:0@lo:400:kworker.0 00000100:00080000:0.0:1767320972.232070:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320972.232074:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320972.232077:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983204480:0@lo:38:kworker.0 00000100:00080000:0.0:1767320972.232087:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320972.232091:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320972.232093:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983204736:0@lo:38:kworker.0 00000100:00080000:0.0:1767320972.232100:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320972.232102:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320972.232105:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983204864:0@lo:38:kworker.0 00000100:00100000:2.0:1767320972.232573:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163983205248/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767320972.232758:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983205248:0@lo:38:kworker.0 00000100:00100000:0.0:1767320972.232808:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320972.235853:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205248 00000100:00100000:1.0:1767320972.235905:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205248 00000100:00100000:1.0:1767320972.235975:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2695 00000100:00100000:1.0:1767320972.235983:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983205248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320972.236006:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320972.236263:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983205248:12345-0@lo:38:kworker.0 Request processed in 279us (3453us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320972.236304:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2695 00000100:00080000:2.0:1767320972.236637:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320972.236644:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320972.236648:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983205248:0@lo:38:kworker.0 00010000:00010000:3.0:1767320972.420696:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320973.446284:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320974.220049:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767320974.220192:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320974.220408:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711744 00000100:00100000:0.0:1767320974.220418:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711744 00000100:00100000:0.0:1767320974.220496:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711616 00000100:00100000:0.0:1767320974.220499:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711616 00000100:00100000:0.0:1767320974.220521:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2036 00000100:00100000:0.0:1767320974.220531:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915711744:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320974.220621:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915711744:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (583us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320974.220629:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2036 00000100:00100000:0.0:1767320974.220648:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2696 00000100:00100000:0.0:1767320974.220652:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915711616:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320974.220670:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320974.220695:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915711616:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (504us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320974.220700:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2696 00000100:00080000:1.0:1767320974.221168:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320974.468448:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320974.788614:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320974.788642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983205376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320974.788722:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320974.788823:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205376 00000100:00100000:2.0:1767320974.788832:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205376 00000100:00100000:2.0:1767320974.788915:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1358 00000100:00100000:2.0:1767320974.788925:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983205376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320974.788958:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320974.789008:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983205376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 84us (286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320974.789015:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1358 00000100:00080000:0.0:1767320974.789126:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320974.789184:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983205504:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320974.789216:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320974.789482:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205504 00000100:00100000:0.0:1767320974.789487:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205504 00000100:00100000:0.0:1767320974.789531:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1359 00000100:00100000:0.0:1767320974.789539:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983205504:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320974.789555:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320974.789593:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983205504:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320974.789598:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1359 00000100:00100000:0.0:1767320974.789639:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983205504:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767320974.789699:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767320974.789802:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983205376:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767320975.493593:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320976.516990:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767320977.348692:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320977.348703:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767320977.348769:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320977.348774:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320977.348777:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320977.348786:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6220 00000100:00080000:3.0:1767320977.348790:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1767320977.348793:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983205632:0@lo:400:kworker.0 00000100:00080000:3.0:1767320977.348796:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320977.348832:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435bb80 x1853163983205760/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320977.348880:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983205760:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.348900:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320977.348990:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320977.348993:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320977.348996:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320977.349008:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6220 00000100:00080000:3.0:1767320977.349010:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320977.349015:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320977.349027:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359f80 x1853163983206016/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320977.349049:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320977.349052:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320977.349054:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320977.349057:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6220 00000100:00080000:3.0:1767320977.349059:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320977.349071:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320977.349082:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359c00 x1853163983206144/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767320977.349102:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767320977.349105:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767320977.349106:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767320977.349109:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6220 00000100:00080000:3.0:1767320977.349111:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767320977.349114:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767320977.349122:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358000 x1853163983206528/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320977.349144:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983205888:0@lo:400:kworker.0 00000100:00100000:1.0:1767320977.349206:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983206272:0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.349432:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.349461:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983206016:0@lo:38:kworker.0 00000100:00100000:2.0:1767320977.349482:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.349499:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983206144:0@lo:38:kworker.0 00000100:00100000:2.0:1767320977.349534:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.349578:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983206528:0@lo:38:kworker.0 00000100:00100000:2.0:1767320977.349598:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320977.349642:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205632 00000100:00100000:2.0:1767320977.349649:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205632 00000100:00100000:2.0:1767320977.349714:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205760 00000100:00100000:2.0:1767320977.349717:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205760 00000100:00100000:2.0:1767320977.349725:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206016 00000100:00100000:2.0:1767320977.349727:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206016 00000100:00100000:2.0:1767320977.349733:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206528 00000100:00100000:2.0:1767320977.349735:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206528 00000100:00100000:2.0:1767320977.349744:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037 00000100:00100000:2.0:1767320977.349753:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983205632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.349823:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983205632:12345-0@lo:400:kworker.0 Request processed in 70us (922us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320977.349831:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037 00000100:00100000:2.0:1767320977.349847:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2697 00000100:00100000:2.0:1767320977.349852:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983205760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320977.349869:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320977.349902:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983205760:12345-0@lo:38:kworker.0 Request processed in 48us (470us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320977.349907:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2697 00000100:00100000:2.0:1767320977.349918:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2698 00000100:00100000:2.0:1767320977.349921:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983206016:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320977.349930:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320977.349952:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983206016:12345-0@lo:38:kworker.0 Request processed in 29us (468us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320977.349957:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2698 00000100:00100000:2.0:1767320977.349967:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2699 00000100:00100000:2.0:1767320977.349971:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983206528:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320977.349980:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320977.350486:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983206528:12345-0@lo:38:kworker.0 Request processed in 512us (886us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320977.350494:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2699 00000100:00080000:2.0:1767320977.350513:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320977.350544:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320977.350549:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320977.350552:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983206016:0@lo:38:kworker.0 00000100:00080000:2.0:1767320977.350562:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320977.350565:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320977.350567:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983206528:0@lo:38:kworker.0 00000100:00080000:2.0:1767320977.350574:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320977.350577:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320977.350580:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983205760:0@lo:38:kworker.0 00010000:00080000:0.0:1767320977.350678:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b480 x1853163983205632/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320993 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767320977.350691:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983205632:0@lo:400:kworker.0 00000100:00100000:0.0:1767320977.350714:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767320977.350721:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983206400:0@lo:400:kworker.0 00000100:00100000:2.0:1767320977.351130:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206144 00000100:00100000:2.0:1767320977.351137:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206144 00000100:00100000:2.0:1767320977.351167:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 699 00000100:00100000:2.0:1767320977.351172:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983206144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320977.351185:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767320977.351223:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983206144:12345-0@lo:38:kworker.0 Request processed in 49us (1688us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320977.351229:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 699 00000100:00080000:2.0:1767320977.351256:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767320977.351260:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767320977.351263:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983206144:0@lo:38:kworker.0 00000100:00100000:0.0:1767320977.351309:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.351354:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206400 00000100:00100000:0.0:1767320977.351358:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206400 00000100:00100000:0.0:1767320977.351414:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1247 00000100:00100000:0.0:1767320977.351422:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604770050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983206400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320977.351462:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604770050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983206400:12345-0@lo:400:kworker.0 Request processed in 42us (153us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320977.351468:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1247 00000100:00100000:3.0:1767320977.351470:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320977.351522:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983205888 00000100:00100000:3.0:1767320977.351526:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983205888 00000100:00100000:3.0:1767320977.351550:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2038 00000100:00100000:3.0:1767320977.351555:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983205888:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320977.351585:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983205888:12345-0@lo:400:kworker.0 Request processed in 30us (115us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320977.351593:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2038 00010000:00080000:3.0:1767320977.351618:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435a300 x1853163983205888/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320993 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320977.351627:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983205888:0@lo:400:kworker.0 00000100:00080000:3.0:1767320977.351647:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320977.351650:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767320977.351724:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983206400:0@lo:400:kworker.0 00000100:00100000:1.0:1767320977.352184:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320977.352442:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206272 00000100:00100000:0.0:1767320977.352447:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206272 00000100:00100000:0.0:1767320977.352491:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1248 00000100:00100000:0.0:1767320977.352497:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983206272:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320977.352539:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983206272:12345-0@lo:400:kworker.0 Request processed in 42us (356us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320977.352545:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1248 00000100:00100000:2.0:1767320977.352606:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983206272:0@lo:400:kworker.0 00000100:00080000:3.0:1767320977.352683:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767320977.540697:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320978.564518:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320979.339822:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320979.340353:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712000 00000100:00100000:3.0:1767320979.340365:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712000 00000100:00100000:3.0:1767320979.340515:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2039 00000100:00100000:3.0:1767320979.340527:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915712000:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320979.340620:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915712000:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 95us (812us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320979.340628:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2039 00000100:00100000:0.0:1767320979.342177:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320979.342360:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915711872 00000100:00100000:3.0:1767320979.342370:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915711872 00000100:00080000:2.0:1767320979.342394:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320979.342419:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2700 00000100:00100000:3.0:1767320979.342427:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915711872:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320979.342458:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320979.342523:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915711872:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 96us (352us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320979.342531:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2700 00010000:00010000:2.0:1767320979.588581:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320979.908726:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767320979.908750:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983206656:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320979.908822:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320979.908993:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206656 00000100:00100000:2.0:1767320979.909001:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206656 00000100:00100000:2.0:1767320979.909070:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1360 00000100:00100000:2.0:1767320979.909090:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983206656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767320979.909122:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320979.909181:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983206656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 103us (358us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320979.909189:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1360 00000100:00080000:3.0:1767320979.909323:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767320979.910385:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983206656:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320979.910568:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983206784:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320979.910617:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320979.910705:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206784 00000100:00100000:1.0:1767320979.910711:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206784 00000100:00100000:1.0:1767320979.910768:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1361 00000100:00100000:1.0:1767320979.910777:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983206784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767320979.910806:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320979.910856:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983206784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (237us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320979.910867:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1361 00000100:00100000:0.0:1767320979.910895:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983206784:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767320979.910931:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767320980.616144:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320981.636615:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320982.468754:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320982.468763:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320982.468822:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320982.468827:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320982.468830:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320982.468839:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6225 00000100:00080000:2.0:1767320982.468845:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320982.468853:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320982.468876:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a2680 x1853163983207040/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320982.468882:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320982.468897:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983206912:0@lo:400:kworker.0 00000100:00080000:2.0:1767320982.468905:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320982.468908:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320982.468909:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320982.468913:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6225 00000100:00080000:2.0:1767320982.468915:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320982.468919:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320982.468929:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0a80 x1853163983207296/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320982.468936:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320982.468939:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320982.468941:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320982.468944:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6225 00000100:00080000:2.0:1767320982.468946:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320982.468949:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320982.468958:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3b80 x1853163983207424/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320982.468962:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767320982.468977:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320982.468980:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320982.468981:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320982.468985:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6225 00000100:00100000:3.0:1767320982.469017:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983207680:0@lo:400:kworker.0 00000100:00080000:2.0:1767320982.469035:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320982.469038:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320982.469049:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1880 x1853163983207808/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320982.469055:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320982.469072:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983207168:0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.469088:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207680 00000100:00100000:3.0:1767320982.469095:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207680 00000100:00100000:2.0:1767320982.469097:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320982.469117:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767320982.469122:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983207552:0@lo:400:kworker.0 00000100:00100000:2.0:1767320982.469151:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320982.469156:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207552 00000100:00100000:3.0:1767320982.469160:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207552 00000100:00100000:3.0:1767320982.469182:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1249 00000100:00100000:3.0:1767320982.469190:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983207680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320982.469197:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983206912 00000100:00100000:2.0:1767320982.469200:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983206912 00000100:00100000:2.0:1767320982.469224:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207168 00000100:00100000:2.0:1767320982.469227:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207168 00000100:00100000:3.0:1767320982.469236:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983207680:12345-0@lo:400:kworker.0 Request processed in 47us (180us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320982.469243:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1249 00000100:00100000:2.0:1767320982.469245:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040 00000100:00100000:2.0:1767320982.469264:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983206912:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.469295:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1250 00000100:00100000:3.0:1767320982.469302:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983207552:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320982.469312:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983206912:12345-0@lo:400:kworker.0 Request processed in 50us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320982.469319:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040 00000100:00100000:3.0:1767320982.469331:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983207552:12345-0@lo:400:kworker.0 Request processed in 30us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320982.469332:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2041 00000100:00100000:2.0:1767320982.469338:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983207168:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.469338:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1250 00000100:00080000:3.0:1767320982.469354:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767320982.469359:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767320982.469361:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320982.469389:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983207552:0@lo:400:kworker.0 00010000:00080000:3.0:1767320982.469401:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a0e00 x1853163983207168/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320998 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320982.469409:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983207168:0@lo:400:kworker.0 00000100:00100000:3.0:1767320982.469437:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983207680:0@lo:400:kworker.0 00010000:00080000:3.0:1767320982.469445:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1c00 x1853163983206912/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767320998 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320982.469453:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983206912:0@lo:400:kworker.0 00000100:00100000:0.0:1767320982.469479:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983207040:0@lo:38:kworker.0 00000100:00100000:0.0:1767320982.472947:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320982.473080:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983207296:0@lo:38:kworker.0 00000100:00100000:0.0:1767320982.473120:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320982.473218:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983207168:12345-0@lo:400:kworker.0 Request processed in 3877us (4116us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320982.473229:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2041 00000100:00100000:0.0:1767320982.474088:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983207424:0@lo:38:kworker.0 00000100:00100000:0.0:1767320982.474131:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320982.474169:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983207808:0@lo:38:kworker.0 00000100:00100000:0.0:1767320982.474404:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320982.475760:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207040 00000100:00100000:1.0:1767320982.475771:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207040 00000100:00100000:1.0:1767320982.475820:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207296 00000100:00100000:1.0:1767320982.475823:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207296 00000100:00100000:1.0:1767320982.475844:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2701 00000100:00100000:1.0:1767320982.475851:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983207040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320982.475871:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320982.475932:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983207040:12345-0@lo:38:kworker.0 Request processed in 82us (2994us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320982.475939:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2701 00000100:00100000:1.0:1767320982.475954:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2702 00000100:00100000:1.0:1767320982.475958:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983207296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320982.475966:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320982.475988:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983207296:12345-0@lo:38:kworker.0 Request processed in 28us (2867us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320982.475993:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2702 00000100:00100000:1.0:1767320982.476035:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207424 00000100:00100000:1.0:1767320982.476039:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207424 00000100:00100000:1.0:1767320982.476058:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 700 00000100:00100000:1.0:1767320982.476062:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983207424:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320982.476071:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320982.476095:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983207424:12345-0@lo:38:kworker.0 Request processed in 31us (1964us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320982.476100:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 700 00000100:00080000:0.0:1767320982.480100:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320982.480114:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320982.480121:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983207040:0@lo:38:kworker.0 00000100:00080000:0.0:1767320982.480146:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320982.480151:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320982.480154:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983207296:0@lo:38:kworker.0 00000100:00080000:0.0:1767320982.480167:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320982.480171:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320982.480173:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983207424:0@lo:38:kworker.0 00000100:00100000:3.0:1767320982.481944:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207808 00000100:00100000:3.0:1767320982.481958:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207808 00000100:00100000:3.0:1767320982.482079:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2703 00000100:00100000:3.0:1767320982.482088:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983207808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767320982.482112:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320982.482217:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983207808:12345-0@lo:38:kworker.0 Request processed in 129us (7811us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320982.482225:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2703 00000100:00080000:0.0:1767320982.482315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320982.482321:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320982.482325:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983207808:0@lo:38:kworker.0 00010000:00010000:2.0:1767320982.660548:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320983.687686:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320984.462403:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320984.462503:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320984.462932:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712128 00000100:00100000:3.0:1767320984.462943:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712128 00000100:00100000:3.0:1767320984.463058:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712256 00000100:00100000:3.0:1767320984.463062:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712256 00000100:00100000:0.0:1767320984.463134:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2042 00000100:00100000:0.0:1767320984.463158:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915712256:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767320984.463257:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915712256:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 101us (752us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320984.463265:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2042 00000100:00100000:0.0:1767320984.463331:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2704 00000100:00100000:0.0:1767320984.463336:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915712128:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320984.463357:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320984.463382:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915712128:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (992us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320984.463388:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2704 00000100:00080000:2.0:1767320984.463431:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320984.710710:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320985.029406:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983208064:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767320985.029502:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320985.029672:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208064 00000100:00100000:0.0:1767320985.029681:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208064 00000100:00100000:0.0:1767320985.029744:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1362 00000100:00100000:0.0:1767320985.029753:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983208064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320985.029795:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320985.029840:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983208064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (338us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320985.029848:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1362 00000100:00080000:2.0:1767320985.030212:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320985.030251:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983208064:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767320985.030605:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320985.030619:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983207936:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320985.030661:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320985.030800:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983207936 00000100:00100000:0.0:1767320985.030806:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983207936 00000100:00100000:0.0:1767320985.030890:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1363 00000100:00100000:0.0:1767320985.030899:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983207936:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320985.030932:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320985.030988:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983207936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (325us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320985.030995:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1363 00000100:00100000:3.0:1767320985.031594:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983207936:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767320985.035215:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767320985.732500:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320986.760578:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767320987.600610:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.600622:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767320987.600702:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.600706:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.600709:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.600725:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.600730:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.600737:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320987.600761:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604580700 x1853163983208320/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.600811:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.600814:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.600816:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.600820:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.600822:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.600826:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320987.600840:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582300 x1853163983208576/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.600849:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.600852:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.600854:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.600857:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.600860:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.600863:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320987.600876:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583b80 x1853163983208704/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767320987.600897:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767320987.600901:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767320987.600902:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767320987.600906:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6231 00000100:00080000:1.0:1767320987.600908:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767320987.600911:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767320987.600921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163983209088/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320987.600953:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983208448:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.601020:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.601090:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767320987.601097:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983208832:0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.601139:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320987.601207:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208448 00000100:00100000:1.0:1767320987.601215:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208448 00000100:00100000:1.0:1767320987.601268:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043 00000100:00100000:1.0:1767320987.601317:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983208448:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320987.601370:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983208448:12345-0@lo:400:kworker.0 Request processed in 55us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320987.601377:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043 00010000:00080000:1.0:1767320987.601419:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604580380 x1853163983208448/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321003 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320987.601431:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983208448:0@lo:400:kworker.0 00000100:00080000:2.0:1767320987.601568:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767320987.601585:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983208192:0@lo:400:kworker.0 00000100:00100000:2.0:1767320987.601616:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320987.601650:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767320987.601655:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983208960:0@lo:400:kworker.0 00000100:00100000:2.0:1767320987.601676:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320987.601759:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208832 00000100:00100000:0.0:1767320987.601769:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208832 00000100:00100000:0.0:1767320987.601810:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208960 00000100:00100000:0.0:1767320987.601813:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208960 00000100:00100000:0.0:1767320987.601834:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1251 00000100:00100000:0.0:1767320987.601841:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983208832:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320987.601882:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983208832:12345-0@lo:400:kworker.0 Request processed in 41us (742us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320987.601888:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1251 00000100:00100000:0.0:1767320987.601900:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1252 00000100:00100000:0.0:1767320987.601906:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983208960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320987.601934:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983208960:12345-0@lo:400:kworker.0 Request processed in 28us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320987.601940:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1252 00000100:00100000:0.0:1767320987.601977:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983208832:0@lo:400:kworker.0 00000100:00100000:0.0:1767320987.601995:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208192 00000100:00100000:0.0:1767320987.601998:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208192 00000100:00100000:0.0:1767320987.602021:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2044 00000100:00100000:0.0:1767320987.602026:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983208192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767320987.602056:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983208192:12345-0@lo:400:kworker.0 Request processed in 30us (439us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320987.602062:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2044 00000100:00100000:0.0:1767320987.602081:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983208320:0@lo:38:kworker.0 00000100:00100000:0.0:1767320987.602105:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320987.602141:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983208576:0@lo:38:kworker.0 00000100:00100000:0.0:1767320987.602172:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320987.602190:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983208704:0@lo:38:kworker.0 00000100:00100000:0.0:1767320987.602213:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320987.602235:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983209088:0@lo:38:kworker.0 00000100:00100000:0.0:1767320987.602254:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320987.602308:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208320 00000100:00100000:0.0:1767320987.602312:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208320 00000100:00100000:0.0:1767320987.602325:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208576 00000100:00100000:0.0:1767320987.602327:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208576 00000100:00100000:0.0:1767320987.602334:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209088 00000100:00100000:0.0:1767320987.602336:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209088 00000100:00100000:0.0:1767320987.602345:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2705 00000100:00100000:0.0:1767320987.602349:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983208320:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320987.602367:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.602400:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983208320:12345-0@lo:38:kworker.0 Request processed in 50us (294us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.602406:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2705 00000100:00100000:0.0:1767320987.602417:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2706 00000100:00100000:0.0:1767320987.602422:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983208576:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320987.602429:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.602447:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983208576:12345-0@lo:38:kworker.0 Request processed in 25us (276us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.602452:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2706 00000100:00100000:0.0:1767320987.602462:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2707 00000100:00100000:0.0:1767320987.602465:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983209088:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320987.602473:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.602492:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a37100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983209088:12345-0@lo:38:kworker.0 Request processed in 26us (238us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.602497:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2707 00000100:00100000:0.0:1767320987.602513:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983208704 00000100:00100000:0.0:1767320987.602517:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983208704 00000100:00100000:0.0:1767320987.602534:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 701 00000100:00100000:0.0:1767320987.602539:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983208704:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320987.602548:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320987.602570:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983208704:12345-0@lo:38:kworker.0 Request processed in 30us (357us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320987.602575:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 701 00000100:00080000:0.0:1767320987.602602:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.602606:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.602609:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604580700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983208320:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.602617:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.602619:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.602622:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983208576:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.602630:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.602632:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.602635:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983208704:0@lo:38:kworker.0 00000100:00080000:0.0:1767320987.602642:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320987.602644:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320987.602646:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983209088:0@lo:38:kworker.0 00000100:00080000:2.0:1767320987.602855:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320987.602860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767320987.602862:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320987.602883:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983208960:0@lo:400:kworker.0 00010000:00080000:2.0:1767320987.602895:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604582d80 x1853163983208192/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321003 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320987.602905:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983208192:0@lo:400:kworker.0 00010000:00010000:3.0:1767320987.783626:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320988.805838:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1767320989.583422:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915712384 offset 0 length 520: 4 00000100:00100000:1.0:1767320989.583490:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320989.583895:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712512 00000100:00100000:3.0:1767320989.583911:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712512 00000100:00100000:3.0:1767320989.583964:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2708 00000100:00100000:3.0:1767320989.583973:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915712512:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320989.584013:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320989.584080:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915712512:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 107us (590us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320989.584087:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2708 00000100:00100000:2.0:1767320989.586728:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320989.586836:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712640 00000100:00100000:3.0:1767320989.586846:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712640 00000100:00100000:3.0:1767320989.587025:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2045 00000100:00100000:3.0:1767320989.587036:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915712640:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320989.587153:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915712640:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 118us (645us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320989.587161:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2045 00000100:00080000:2.0:1767320989.587205:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767320989.828601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767320990.148687:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320990.148712:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983209216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320990.148722:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767320990.148736:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983209344:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320990.148785:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320990.148834:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209216 00000100:00100000:0.0:1767320990.148842:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209216 00000100:00100000:1.0:1767320990.148862:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320990.148899:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209344 00000100:00100000:1.0:1767320990.148903:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209344 00000100:00100000:0.0:1767320990.148916:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1364 00000100:00100000:0.0:1767320990.148927:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983209216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767320990.148959:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767320990.149006:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983209216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (221us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320990.149014:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1364 00000100:00100000:0.0:1767320990.149034:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1365 00000100:00100000:0.0:1767320990.149053:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983209344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767320990.149070:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:2.0:1767320990.149328:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767320990.149334:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320990.149339:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983209216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767320990.150454:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983209344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1409us (1586us total) trans 0 rc 0/0 00000100:00100000:0.0:1767320990.150464:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1365 00000100:00100000:1.0:1767320990.150622:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a34380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983209344:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320990.853058:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767320991.876500:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320992.708441:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320992.708451:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320992.708505:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320992.708509:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320992.708513:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320992.708521:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6236 00000100:00080000:2.0:1767320992.708526:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320992.708567:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320992.708588:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0e00 x1853163983209600/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320992.708620:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320992.708624:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320992.708625:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320992.708629:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6236 00000100:00080000:2.0:1767320992.708631:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320992.708635:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320992.708644:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163983209856/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320992.708653:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320992.708656:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320992.708658:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320992.708661:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6236 00000100:00080000:2.0:1767320992.708663:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320992.708666:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320992.708676:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1c00 x1853163983209984/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320992.708702:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320992.708706:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320992.708707:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320992.708710:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6236 00000100:00080000:2.0:1767320992.708712:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320992.708715:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320992.708724:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f2680 x1853163983210368/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767320992.708814:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320967/real 1767320967] req@ffff93463f963100 x1853163983202944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320992 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767320992.708843:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463f963100 x1853163983202944/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767320992 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320992.708871:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983209472:0@lo:400:kworker.0 00000100:00100000:3.0:1767320992.708871:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983209600:0@lo:38:kworker.0 00000100:00100000:2.0:1767320992.708958:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983210112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.708998:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983210240:0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.709114:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.709172:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983209728:0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.709193:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.709220:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209472 00000100:00100000:1.0:1767320992.709227:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209472 00000100:00100000:1.0:1767320992.709286:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209728 00000100:00100000:1.0:1767320992.709289:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209728 00000100:00100000:1.0:1767320992.709308:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046 00000100:00100000:1.0:1767320992.709315:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983209472:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.709355:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983209472:12345-0@lo:400:kworker.0 Request processed in 41us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320992.709362:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046 00000100:00100000:1.0:1767320992.709377:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2047 00000100:00100000:1.0:1767320992.709382:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983209728:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.709405:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983209728:12345-0@lo:400:kworker.0 Request processed in 24us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320992.709423:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2047 00010000:00080000:1.0:1767320992.709451:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1180 x1853163983209728/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321008 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320992.709463:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983209728:0@lo:400:kworker.0 00010000:00080000:1.0:1767320992.709484:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a2d80 x1853163983209472/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321008 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320992.709492:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983209472:0@lo:400:kworker.0 00000100:00080000:1.0:1767320992.709523:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767320992.716109:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.716165:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209600 00000100:00100000:1.0:1767320992.716176:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209600 00000100:00100000:3.0:1767320992.716184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983209856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320992.716222:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2709 00000100:00100000:2.0:1767320992.716223:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320992.716235:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983209600:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767320992.716235:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:1.0:1767320992.716262:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320992.716265:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983209984:0@lo:38:kworker.0 00000100:00100000:2.0:1767320992.716308:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209856 00000100:00100000:2.0:1767320992.716313:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209856 00000100:00100000:1.0:1767320992.716321:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983209600:12345-0@lo:38:kworker.0 Request processed in 90us (219us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320992.716328:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2709 00000100:00100000:2.0:1767320992.716332:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2710 00000100:00100000:2.0:1767320992.716337:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983209856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767320992.716347:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320992.716362:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210112 00000100:00100000:1.0:1767320992.716366:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210112 00000100:00100000:2.0:1767320992.716372:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983209856:12345-0@lo:38:kworker.0 Request processed in 35us (141us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767320992.716377:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2710 00000100:00100000:1.0:1767320992.716402:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1253 00000100:00100000:1.0:1767320992.716408:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983210112:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320992.716552:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983210112:12345-0@lo:400:kworker.0 Request processed in 143us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320992.716558:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1253 00000100:00080000:1.0:1767320992.716574:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320992.717151:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983210112:0@lo:400:kworker.0 00000100:00100000:0.0:1767320992.718443:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320992.719049:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767320992.719113:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983210368:0@lo:38:kworker.0 00000100:00100000:3.0:1767320992.719148:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767320992.719172:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320992.719188:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767320992.719194:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983202944:0@lo:250:kworker.0 00000100:00080000:3.0:1767320992.719234:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320992.719239:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320992.719242:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983209600:0@lo:38:kworker.0 00000100:00080000:3.0:1767320992.719254:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320992.719257:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320992.719260:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983209856:0@lo:38:kworker.0 00000100:00100000:1.0:1767320992.719366:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983209984 00000100:00100000:1.0:1767320992.719374:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983209984 00000100:00100000:1.0:1767320992.719488:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 702 00000100:00100000:1.0:1767320992.719496:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983209984:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767320992.719515:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767320992.719556:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983209984:12345-0@lo:38:kworker.0 Request processed in 60us (508us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767320992.719563:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 702 00000100:00100000:0.0:1767320992.719677:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210368 00000100:00100000:0.0:1767320992.719682:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210368 00000100:00100000:0.0:1767320992.719709:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2711 00000100:00100000:0.0:1767320992.719713:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983210368:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320992.719724:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320992.719751:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983210368:12345-0@lo:38:kworker.0 Request processed in 36us (602us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320992.719756:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2711 00000100:00080000:0.0:1767320992.719774:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767320992.719778:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767320992.719781:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767320992.719785:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6215 00000100:00080000:0.0:1767320992.719789:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767320992.719794:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320992.719817:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210240 00000100:00100000:2.0:1767320992.719823:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210240 00000100:00100000:2.0:1767320992.719867:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1254 00000100:00100000:2.0:1767320992.719874:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983210240:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320992.719923:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983210240:12345-0@lo:400:kworker.0 Request processed in 49us (1482us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320992.719931:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1254 00000100:00080000:3.0:1767320992.719941:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320992.719947:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00080000:1.0:1767320992.720008:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767320992.720040:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983210240:0@lo:400:kworker.0 00000100:00100000:3.0:1767320992.720392:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983209984:0@lo:38:kworker.0 00000100:00080000:3.0:1767320992.720406:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767320992.720410:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767320992.720415:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983210368:0@lo:38:kworker.0 00000100:00100000:0.0:1767320992.720712:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631988e00 x1853163983210496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767320992.720776:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983210496:0@lo:250:kworker.0 00000400:00000100:1.0:1767320992.720808:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983210496 offset 0 length 520: 4 00010000:00010000:3.0:1767320992.902310:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320993.927460:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767320994.699828:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320994.699883:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767320994.699984:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712768 00000100:00100000:3.0:1767320994.699996:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712768 00000100:00100000:3.0:1767320994.700052:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915712896 00000100:00100000:3.0:1767320994.700056:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915712896 00000100:00100000:3.0:1767320994.700099:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2048 00000100:00100000:3.0:1767320994.700109:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915712896:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767320994.700180:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915712896:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767320994.700188:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2048 00000100:00100000:3.0:1767320994.700206:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2712 00000100:00100000:3.0:1767320994.700211:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915712768:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767320994.700231:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767320994.700255:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915712768:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (439us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767320994.700261:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2712 00000100:00080000:3.0:1767320994.700311:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767320994.950761:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767320995.268698:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767320995.268752:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983210624:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767320995.268865:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320995.268902:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210624 00000100:00100000:1.0:1767320995.268923:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210624 00000100:00100000:1.0:1767320995.269028:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1366 00000100:00100000:1.0:1767320995.269038:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983210624:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767320995.269089:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767320995.269135:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983210624:12345-0@lo:13:osp-pre-1-1.0 Request processed in 99us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320995.269143:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1366 00000100:00080000:3.0:1767320995.269248:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767320995.269314:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983210624:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767320995.269408:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983210752:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767320995.269437:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320995.269529:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210752 00000100:00100000:2.0:1767320995.269534:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210752 00000100:00080000:0.0:1767320995.269561:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767320995.269574:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1367 00000100:00100000:2.0:1767320995.269582:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983210752:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767320995.269599:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767320995.273950:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983210752:12345-0@lo:13:osp-pre-0-1.0 Request processed in 4355us (4498us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320995.273966:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1367 00000100:00100000:1.0:1767320995.280224:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983210752:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767320995.975392:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320996.996518:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767320997.829624:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320997.829635:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767320997.829713:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320997.829717:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320997.829721:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320997.829729:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6241 00000100:00080000:2.0:1767320997.829734:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320997.829740:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320997.829788:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462dee7b80 x1853163983211008/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320997.829858:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320997.829861:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320997.829864:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320997.829868:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6241 00000100:00080000:2.0:1767320997.829870:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320997.829885:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320997.829897:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396ca80 x1853163983211264/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320997.829906:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320997.829909:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320997.829910:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320997.829914:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6241 00000100:00080000:2.0:1767320997.829916:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320997.829920:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320997.829938:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163983211392/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767320997.829956:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767320997.829959:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767320997.829970:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767320997.829973:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6241 00000100:00080000:2.0:1767320997.829976:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767320997.829979:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767320997.829999:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983211776/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767320997.830027:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983211520:0@lo:400:kworker.0 00000100:00100000:2.0:1767320997.830103:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320997.830158:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767320997.830164:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983211136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320997.830189:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767320997.830229:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211520 00000100:00100000:2.0:1767320997.830236:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211520 00000100:00100000:2.0:1767320997.830312:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1255 00000100:00100000:2.0:1767320997.830321:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983211520:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320997.830373:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983211520:12345-0@lo:400:kworker.0 Request processed in 52us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320997.830380:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1255 00000100:00100000:2.0:1767320997.830402:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211136 00000100:00100000:2.0:1767320997.830406:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211136 00000100:00100000:2.0:1767320997.830431:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2049 00000100:00100000:2.0:1767320997.830438:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983211136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767320997.830468:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983211136:12345-0@lo:400:kworker.0 Request processed in 30us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320997.830474:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2049 00000100:00080000:2.0:1767320997.830491:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767320997.830495:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00080000:2.0:1767320997.830526:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396d500 x1853163983211136/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321013 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767320997.830538:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983211136:0@lo:400:kworker.0 00000100:00100000:2.0:1767320997.830549:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983211520:0@lo:400:kworker.0 00000100:00100000:0.0:1767320997.831079:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983211008:0@lo:38:kworker.0 00000100:00100000:0.0:1767320997.831117:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320997.831158:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983211264:0@lo:38:kworker.0 00000100:00100000:0.0:1767320997.831180:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320997.831199:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983211392:0@lo:38:kworker.0 00000100:00100000:0.0:1767320997.831244:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320997.831263:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983211776:0@lo:38:kworker.0 00000100:00100000:0.0:1767320997.831319:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767320997.831362:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211008 00000100:00100000:0.0:1767320997.831367:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211008 00000100:00100000:0.0:1767320997.831399:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211264 00000100:00100000:0.0:1767320997.831403:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211264 00000100:00100000:0.0:1767320997.831411:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211776 00000100:00100000:0.0:1767320997.831413:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211776 00000100:00100000:0.0:1767320997.831423:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2713 00000100:00100000:0.0:1767320997.831429:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320997.831450:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320997.831484:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211008:12345-0@lo:38:kworker.0 Request processed in 54us (366us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320997.831490:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2713 00000100:00100000:0.0:1767320997.831521:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2714 00000100:00100000:0.0:1767320997.831525:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211264:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320997.831533:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320997.831556:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604321f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211264:12345-0@lo:38:kworker.0 Request processed in 30us (375us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320997.831562:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2714 00000100:00100000:0.0:1767320997.831573:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2715 00000100:00100000:0.0:1767320997.831576:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320997.831586:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320997.831606:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983211776:12345-0@lo:38:kworker.0 Request processed in 29us (287us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320997.831612:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2715 00000100:00100000:0.0:1767320997.831629:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211392 00000100:00100000:0.0:1767320997.831632:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211392 00000100:00100000:0.0:1767320997.831651:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 703 00000100:00100000:0.0:1767320997.831656:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983211392:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767320997.831664:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320997.831689:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983211392:12345-0@lo:38:kworker.0 Request processed in 32us (445us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320997.831693:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 703 00000100:00080000:0.0:1767320997.831729:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320997.831735:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320997.831738:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983211008:0@lo:38:kworker.0 00000100:00080000:0.0:1767320997.831747:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320997.831750:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320997.831753:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983211264:0@lo:38:kworker.0 00000100:00080000:0.0:1767320997.831760:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320997.831763:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320997.831765:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983211392:0@lo:38:kworker.0 00000100:00080000:0.0:1767320997.831771:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767320997.831774:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767320997.831777:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983211776:0@lo:38:kworker.0 00000100:00100000:1.0:1767320997.832110:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983211648:0@lo:400:kworker.0 00000100:00100000:1.0:1767320997.832163:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320997.832191:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767320997.832197:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983210880:0@lo:400:kworker.0 00000100:00100000:1.0:1767320997.832215:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767320997.832236:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983210880 00000100:00100000:1.0:1767320997.832239:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983210880 00000100:00100000:1.0:1767320997.832292:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2050 00000100:00100000:1.0:1767320997.832299:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983210880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320997.832338:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983210880:12345-0@lo:400:kworker.0 Request processed in 40us (121us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320997.832344:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2050 00000100:00100000:1.0:1767320997.832382:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211648 00000100:00100000:1.0:1767320997.832387:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211648 00000100:00100000:1.0:1767320997.832413:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1256 00000100:00100000:1.0:1767320997.832419:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983211648:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767320997.832447:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983211648:12345-0@lo:400:kworker.0 Request processed in 28us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1767320997.832453:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1256 00000100:00080000:1.0:1767320997.832468:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767320997.832472:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767320997.832492:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee4e00 x1853163983210880/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321013 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767320997.832503:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983210880:0@lo:400:kworker.0 00000100:00100000:1.0:1767320997.832512:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983211648:0@lo:400:kworker.0 00010000:00010000:2.0:1767320998.020724:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767320999.044550:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767320999.820226:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767320999.822142:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713152 00000100:00100000:2.0:1767320999.822158:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713152 00000100:00100000:2.0:1767320999.822322:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2051 00000100:00100000:2.0:1767320999.822333:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915713152:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767320999.822430:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915713152:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 98us (2354us total) trans 0 rc 0/0 00000100:00100000:2.0:1767320999.822439:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2051 00000100:00100000:3.0:1767320999.825445:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767320999.825670:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713024 00000100:00100000:0.0:1767320999.825681:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713024 00000100:00100000:0.0:1767320999.825729:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2716 00000100:00100000:0.0:1767320999.825737:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713024:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767320999.825760:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767320999.825818:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713024:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 80us (385us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767320999.825825:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2716 00000100:00080000:1.0:1767320999.827889:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321000.068706:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321000.390679:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321000.390704:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983211904:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321000.390775:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321000.390988:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983212032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321000.391027:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321000.391216:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983211904 00000100:00100000:2.0:1767321000.391226:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983211904 00000100:00100000:2.0:1767321000.391322:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212032 00000100:00100000:2.0:1767321000.391326:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212032 00000100:00100000:2.0:1767321000.391345:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1368 00000100:00100000:2.0:1767321000.391353:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983211904:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321000.391384:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321000.391430:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983211904:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (654us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321000.391436:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1368 00000100:00100000:2.0:1767321000.391451:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1369 00000100:00100000:2.0:1767321000.391457:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983212032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321000.391473:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321000.391506:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983212032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (478us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321000.391512:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1369 00000100:00100000:2.0:1767321000.391556:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983212032:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321000.391692:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321000.391697:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321000.391722:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983211904:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321001.092502:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321002.118479:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321002.948519:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321002.948532:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321002.948582:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321002.948586:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321002.948767:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321002.948776:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6246 00000100:00080000:2.0:1767321002.948781:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321002.948787:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321002.948811:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd880 x1853163983212288/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321002.948838:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321002.948842:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321002.948844:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321002.948847:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6246 00000100:00080000:2.0:1767321002.948850:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321002.948853:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321002.948862:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983212544/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321002.948884:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321002.948895:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321002.948897:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321002.948900:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6246 00000100:00080000:2.0:1767321002.948903:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321002.948906:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321002.948917:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163983212672/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321002.948938:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321002.948941:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321002.948943:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321002.948946:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6246 00000100:00080000:2.0:1767321002.948948:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321002.948952:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321002.948961:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d500 x1853163983213056/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321002.949003:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983212928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321002.949006:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983212288:0@lo:38:kworker.0 00000100:00100000:3.0:1767321002.949086:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983212800:0@lo:400:kworker.0 00000100:00100000:3.0:1767321002.949136:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321002.949182:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321002.949189:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321002.949194:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983212416:0@lo:400:kworker.0 00000100:00100000:2.0:1767321002.949215:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321002.949223:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983212160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321002.949231:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321002.949297:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212416 00000100:00100000:3.0:1767321002.949305:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212416 00000100:00100000:2.0:1767321002.949358:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321002.949364:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052 00000100:00100000:3.0:1767321002.949376:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983212416:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321002.949468:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212800 00000100:00100000:3.0:1767321002.949471:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212928 00000100:00100000:3.0:1767321002.949476:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212928 00000100:00100000:1.0:1767321002.949484:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212800 00000100:00100000:3.0:1767321002.949509:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1257 00000100:00100000:3.0:1767321002.949518:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983212928:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321002.949526:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1258 00000100:00100000:1.0:1767321002.949537:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983212800:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321002.949561:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983212928:12345-0@lo:400:kworker.0 Request processed in 43us (373us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321002.949570:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1257 00000100:00100000:3.0:1767321002.949628:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983212416:12345-0@lo:400:kworker.0 Request processed in 255us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321002.949634:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052 00000100:00100000:3.0:1767321002.949645:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212160 00000100:00100000:3.0:1767321002.949648:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212160 00000100:00100000:3.0:1767321002.949669:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2053 00000100:00100000:3.0:1767321002.949675:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983212160:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321002.949703:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604377b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983212160:12345-0@lo:400:kworker.0 Request processed in 28us (347us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321002.949709:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2053 00000100:00100000:1.0:1767321002.949757:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983212800:12345-0@lo:400:kworker.0 Request processed in 56us (453us total) trans 0 rc 0/0 00010000:00080000:3.0:1767321002.949760:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382dc000 x1853163983212160/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321018 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321002.949771:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1258 00000100:00100000:3.0:1767321002.949782:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983212160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321002.949798:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983212928:0@lo:400:kworker.0 00000100:00080000:1.0:1767321002.949801:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321002.949805:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321002.949809:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321002.949829:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8700 x1853163983212416/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321018 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321002.949838:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983212416:0@lo:400:kworker.0 00000100:00100000:1.0:1767321002.949847:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983212800:0@lo:400:kworker.0 00000100:00100000:0.0:1767321002.949934:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321002.949956:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983212544:0@lo:38:kworker.0 00000100:00100000:0.0:1767321002.949979:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321002.949995:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983212672:0@lo:38:kworker.0 00000100:00100000:0.0:1767321002.950021:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321002.950046:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983213056:0@lo:38:kworker.0 00000100:00100000:0.0:1767321002.950067:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321002.950094:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212288 00000100:00100000:0.0:1767321002.950098:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212288 00000100:00100000:0.0:1767321002.950111:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212544 00000100:00100000:0.0:1767321002.950113:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212544 00000100:00100000:1.0:1767321002.950125:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213056 00000100:00100000:0.0:1767321002.950129:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2717 00000100:00100000:1.0:1767321002.950132:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213056 00000100:00100000:0.0:1767321002.950137:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983212288:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321002.950155:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2718 00010000:00020000:0.0:1767321002.950157:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321002.950164:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983212544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321002.950176:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321002.950200:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983212288:12345-0@lo:38:kworker.0 Request processed in 61us (265us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321002.950207:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2717 00000100:00100000:1.0:1767321002.950214:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983212544:12345-0@lo:38:kworker.0 Request processed in 49us (233us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321002.950224:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2719 00000100:00100000:0.0:1767321002.950230:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983213056:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321002.950239:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321002.950267:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983213056:12345-0@lo:38:kworker.0 Request processed in 37us (199us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321002.950313:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983212672 00000100:00100000:2.0:1767321002.950318:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983212672 00000100:00100000:2.0:1767321002.950339:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 704 00000100:00100000:2.0:1767321002.950343:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345013f1050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983212672:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321002.950354:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321002.950381:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345013f1050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983212672:12345-0@lo:38:kworker.0 Request processed in 36us (359us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321002.950387:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 704 00000100:00080000:2.0:1767321002.952746:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321002.952762:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321002.952767:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983212288:0@lo:38:kworker.0 00000100:00080000:2.0:1767321002.952780:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321002.952783:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321002.952787:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983212544:0@lo:38:kworker.0 00000100:00080000:2.0:1767321002.952796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321002.952800:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321002.952802:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983212672:0@lo:38:kworker.0 00000100:00080000:2.0:1767321002.952812:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321002.952816:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321002.952818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983213056:0@lo:38:kworker.0 00000100:00100000:1.0:1767321002.955684:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2718 00000100:00100000:0.0:1767321002.955698:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2719 00010000:00010000:3.0:1767321003.140912:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321004.167918:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321004.943664:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321004.943744:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713408 00000100:00100000:1.0:1767321004.943756:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713408 00000100:00100000:1.0:1767321004.943853:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2054 00000100:00100000:1.0:1767321004.943864:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915713408:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321004.943960:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915713408:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 86us (296us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321004.943980:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2054 00000100:00080000:2.0:1767321004.944218:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321004.956372:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321004.956526:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713280 00000100:00100000:3.0:1767321004.956537:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713280 00000100:00100000:3.0:1767321004.956594:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2720 00000100:00100000:3.0:1767321004.956604:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713280:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321004.956630:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321004.956729:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713280:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 124us (368us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321004.956738:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2720 00010000:00010000:3.0:1767321005.188861:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321005.509140:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983213184:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321005.509219:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321005.509613:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983213312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321005.509642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321005.515256:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213184 00000100:00100000:0.0:1767321005.515292:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213184 00000100:00100000:0.0:1767321005.515419:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213312 00000100:00100000:0.0:1767321005.515424:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213312 00000100:00080000:2.0:1767321005.515473:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321005.515478:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321005.516880:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1370 00000100:00100000:1.0:1767321005.516899:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983213184:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321005.516933:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321005.517092:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983213184:12345-0@lo:13:osp-pre-0-1.0 Request processed in 199us (7872us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321005.517099:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1370 00000100:00100000:1.0:1767321005.517119:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1371 00000100:00100000:1.0:1767321005.517126:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983213312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321005.517144:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321005.517181:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983213312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (7537us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321005.517187:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1371 00000100:00100000:3.0:1767321005.517390:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983213312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321005.520567:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983213184:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767321006.213049:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321007.236832:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321008.068690:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321008.068700:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321008.068755:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321008.068759:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321008.068764:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321008.068805:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6251 00000100:00080000:2.0:1767321008.068810:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321008.068817:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321008.068839:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375180 x1853163983213568/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321008.068869:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321008.068873:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321008.068874:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321008.068877:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6251 00000100:00080000:2.0:1767321008.068880:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321008.068883:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321008.068907:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374380 x1853163983213824/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321008.068916:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321008.068919:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321008.068921:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321008.068924:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6251 00000100:00080000:2.0:1767321008.068926:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321008.068930:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321008.068942:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983213952/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321008.068963:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321008.068967:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321008.068969:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321008.068972:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6251 00000100:00080000:2.0:1767321008.068974:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321008.068978:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321008.068987:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b800 x1853163983214336/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321008.069012:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983213696:0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.069076:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321008.069142:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321008.069149:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983214080:0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.069178:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321008.069215:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214080 00000100:00100000:2.0:1767321008.069231:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214080 00000100:00100000:2.0:1767321008.069296:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1259 00000100:00100000:2.0:1767321008.069305:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983214080:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321008.069351:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983214080:12345-0@lo:400:kworker.0 Request processed in 48us (172us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321008.069357:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1259 00000100:00080000:2.0:1767321008.069372:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321008.069397:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983214080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.069700:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983214208:0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.069725:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321008.069749:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321008.069753:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983213440:0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.069767:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321008.069806:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213696 00000100:00100000:3.0:1767321008.069810:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213696 00000100:00100000:3.0:1767321008.069840:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213440 00000100:00100000:3.0:1767321008.069843:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213440 00000100:00100000:3.0:1767321008.069860:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055 00000100:00100000:3.0:1767321008.069866:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983213696:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.069901:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee6680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983213696:12345-0@lo:400:kworker.0 Request processed in 35us (825us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321008.069907:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055 00000100:00100000:3.0:1767321008.069920:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2056 00000100:00100000:3.0:1767321008.069926:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983213440:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321008.069954:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983213440:12345-0@lo:400:kworker.0 Request processed in 29us (185us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321008.069959:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2056 00010000:00080000:3.0:1767321008.069988:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604376680 x1853163983213440/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321024 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321008.070003:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983213440:0@lo:400:kworker.0 00000100:00080000:3.0:1767321008.070018:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767321008.070032:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375500 x1853163983213696/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321024 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321008.070040:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983213696:0@lo:400:kworker.0 00000100:00100000:0.0:1767321008.070801:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983213568:0@lo:38:kworker.0 00000100:00100000:0.0:1767321008.070843:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.070876:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983213824:0@lo:38:kworker.0 00000100:00100000:0.0:1767321008.070894:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.070906:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983213952:0@lo:38:kworker.0 00000100:00100000:0.0:1767321008.070965:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.070981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983214336:0@lo:38:kworker.0 00000100:00100000:0.0:1767321008.070994:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321008.071017:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213952 00000100:00100000:0.0:1767321008.071021:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213952 00000100:00100000:0.0:1767321008.071041:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 705 00000100:00100000:0.0:1767321008.071045:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983213952:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321008.071062:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321008.071098:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983213952:12345-0@lo:38:kworker.0 Request processed in 51us (130us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321008.071104:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 705 00000100:00100000:0.0:1767321008.071120:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213568 00000100:00100000:0.0:1767321008.071122:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213568 00000100:00100000:0.0:1767321008.071129:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983213824 00000100:00100000:0.0:1767321008.071130:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983213824 00000100:00100000:0.0:1767321008.071135:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214336 00000100:00100000:0.0:1767321008.071136:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214336 00000100:00100000:0.0:1767321008.071142:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2721 00000100:00100000:0.0:1767321008.071144:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983213568:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321008.071151:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321008.071167:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983213568:12345-0@lo:38:kworker.0 Request processed in 21us (324us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321008.071170:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2721 00000100:00100000:0.0:1767321008.071176:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2722 00000100:00100000:0.0:1767321008.071178:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983213824:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321008.071183:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321008.071195:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983213824:12345-0@lo:38:kworker.0 Request processed in 16us (300us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321008.071200:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2722 00000100:00100000:0.0:1767321008.071209:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2723 00000100:00100000:0.0:1767321008.071212:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983214336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321008.071220:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321008.071239:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983214336:12345-0@lo:38:kworker.0 Request processed in 26us (243us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321008.071245:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2723 00000100:00080000:0.0:1767321008.071295:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321008.071299:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321008.071301:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983213568:0@lo:38:kworker.0 00000100:00080000:0.0:1767321008.071307:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321008.071309:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321008.071311:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983213824:0@lo:38:kworker.0 00000100:00080000:0.0:1767321008.071315:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321008.071318:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321008.071321:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983213952:0@lo:38:kworker.0 00000100:00080000:0.0:1767321008.071329:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321008.071331:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321008.071334:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983214336:0@lo:38:kworker.0 00000100:00100000:1.0:1767321008.071792:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214208 00000100:00100000:1.0:1767321008.071798:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214208 00000100:00100000:1.0:1767321008.071853:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1260 00000100:00100000:1.0:1767321008.071860:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983214208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321008.071909:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983214208:12345-0@lo:400:kworker.0 Request processed in 49us (2182us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321008.071915:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1260 00000100:00100000:1.0:1767321008.071947:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983214208:0@lo:400:kworker.0 00000100:00080000:3.0:1767321008.072097:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:3.0:1767321008.264268:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321009.287134:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321010.078860:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321010.079179:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321010.080223:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713536 00000100:00100000:2.0:1767321010.080236:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713536 00000100:00100000:2.0:1767321010.080331:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713664 00000100:00100000:2.0:1767321010.080335:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713664 00000100:00080000:0.0:1767321010.080407:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321010.080495:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2057 00000100:00100000:1.0:1767321010.080508:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915713664:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321010.080605:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915713664:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 99us (1425us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321010.080613:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2057 00000100:00100000:1.0:1767321010.080632:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2724 00000100:00100000:1.0:1767321010.080637:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915713536:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321010.080656:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321010.080681:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915713536:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 43us (1831us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321010.080688:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2724 00010000:00010000:2.0:1767321010.309930:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321010.628621:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321010.628652:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983214464:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321010.628769:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321010.628962:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214464 00000100:00100000:1.0:1767321010.628970:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214464 00000100:00100000:1.0:1767321010.629042:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1372 00000100:00100000:1.0:1767321010.629063:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983214464:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321010.629095:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321010.629151:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983214464:12345-0@lo:13:osp-pre-1-1.0 Request processed in 100us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321010.629159:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1372 00000100:00080000:0.0:1767321010.629354:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321010.629495:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983214464:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321010.629753:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321010.629765:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983214592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321010.629800:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321010.629900:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214592 00000100:00100000:1.0:1767321010.629905:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214592 00000100:00100000:1.0:1767321010.629950:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1373 00000100:00100000:1.0:1767321010.629958:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983214592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321010.629980:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321010.630017:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983214592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 59us (215us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321010.630023:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1373 00000100:00100000:2.0:1767321010.630220:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983214592:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321010.630546:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767321011.332546:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321012.356538:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321013.188572:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321013.188583:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321013.188646:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321013.188651:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321013.188654:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321013.188662:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6256 00000100:00080000:0.0:1767321013.188667:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321013.188673:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321013.188706:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cc380 x1853163983214848/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321013.188752:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321013.188755:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321013.188757:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321013.188761:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6256 00000100:00080000:0.0:1767321013.188764:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321013.188767:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321013.188778:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cea00 x1853163983215104/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321013.188787:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321013.188790:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321013.188791:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321013.188795:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6256 00000100:00080000:0.0:1767321013.188797:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321013.188800:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321013.188810:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cdf80 x1853163983215232/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321013.188829:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321013.188832:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321013.188834:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321013.188838:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6256 00000100:00080000:0.0:1767321013.188841:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321013.188844:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321013.188857:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631909c00 x1853163983215616/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321013.189022:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983214848:0@lo:38:kworker.0 00000100:00100000:1.0:1767321013.189101:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189173:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983215104:0@lo:38:kworker.0 00000100:00100000:1.0:1767321013.189195:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189215:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983215232:0@lo:38:kworker.0 00000100:00100000:1.0:1767321013.189240:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189264:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983215616:0@lo:38:kworker.0 00000100:00100000:1.0:1767321013.189313:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189345:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214848 00000100:00100000:1.0:1767321013.189352:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214848 00000100:00100000:1.0:1767321013.189383:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215104 00000100:00100000:1.0:1767321013.189386:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215104 00000100:00100000:1.0:1767321013.189398:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215616 00000100:00100000:1.0:1767321013.189400:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215616 00000100:00100000:1.0:1767321013.189412:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2725 00000100:00100000:1.0:1767321013.189419:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983214848:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321013.189438:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321013.189479:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983214848:12345-0@lo:38:kworker.0 Request processed in 59us (377us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321013.189487:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2725 00000100:00100000:1.0:1767321013.189504:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2726 00000100:00100000:1.0:1767321013.189508:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983215104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321013.189518:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321013.189541:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983215104:12345-0@lo:38:kworker.0 Request processed in 31us (345us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321013.189547:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2726 00000100:00100000:1.0:1767321013.189558:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2727 00000100:00100000:1.0:1767321013.189562:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983215616:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321013.189571:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321013.189590:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983215616:12345-0@lo:38:kworker.0 Request processed in 27us (277us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321013.189596:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2727 00000100:00100000:1.0:1767321013.189613:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215232 00000100:00100000:1.0:1767321013.189617:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215232 00000100:00100000:1.0:1767321013.189635:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767321013.189639:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983215232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321013.189647:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321013.189670:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983215232:12345-0@lo:38:kworker.0 Request processed in 29us (428us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321013.189675:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 706 00000100:00100000:1.0:1767321013.189721:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983215360:0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.189761:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189782:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321013.189789:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983214976:0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.189810:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321013.189832:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214976 00000100:00100000:1.0:1767321013.189837:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214976 00000100:00100000:1.0:1767321013.189871:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2058 00000100:00100000:1.0:1767321013.189878:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983214976:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321013.189918:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983214976:12345-0@lo:400:kworker.0 Request processed in 40us (106us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321013.189924:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2058 00010000:00080000:1.0:1767321013.189958:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059ced80 x1853163983214976/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321013.189970:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983214976:0@lo:400:kworker.0 00000100:00080000:1.0:1767321013.189999:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321013.190004:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321013.190008:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983214848:0@lo:38:kworker.0 00000100:00080000:1.0:1767321013.190016:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321013.190020:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321013.190023:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983215104:0@lo:38:kworker.0 00000100:00080000:1.0:1767321013.190030:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321013.190033:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321013.190036:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983215232:0@lo:38:kworker.0 00000100:00080000:1.0:1767321013.190043:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321013.190046:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321013.190048:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983215616:0@lo:38:kworker.0 00000100:00100000:2.0:1767321013.190376:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983214720:0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190408:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321013.190438:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321013.190445:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ce680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983215488:0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190467:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321013.190496:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983214720 00000100:00100000:2.0:1767321013.190500:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983214720 00000100:00100000:2.0:1767321013.190531:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2059 00000100:00100000:2.0:1767321013.190538:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983214720:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190586:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983214720:12345-0@lo:400:kworker.0 Request processed in 48us (177us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321013.190592:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2059 00000100:00100000:2.0:1767321013.190612:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215360 00000100:00100000:2.0:1767321013.190615:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215360 00000100:00100000:2.0:1767321013.190641:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1261 00000100:00100000:2.0:1767321013.190648:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983215360:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190694:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983215360:12345-0@lo:400:kworker.0 Request processed in 47us (932us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321013.190700:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1261 00000100:00100000:2.0:1767321013.190710:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215488 00000100:00100000:2.0:1767321013.190713:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215488 00000100:00100000:2.0:1767321013.190851:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1262 00000100:00100000:2.0:1767321013.190862:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983215488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190893:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983215488:12345-0@lo:400:kworker.0 Request processed in 34us (426us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321013.190898:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1262 00000100:00100000:2.0:1767321013.190933:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ce680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983215488:0@lo:400:kworker.0 00010000:00080000:2.0:1767321013.190943:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e6f3b80 x1853163983214720/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321029 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321013.190952:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983214720:0@lo:400:kworker.0 00000100:00100000:2.0:1767321013.190978:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983215360:0@lo:400:kworker.0 00000100:00080000:0.0:1767321013.191292:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321013.191297:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321013.191299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321013.383405:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321014.161926:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767321014.161965:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915713792 offset 0 length 520: 4 00000100:00100000:0.0:1767321014.161988:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321014.162092:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714048 00000100:00100000:1.0:1767321014.162104:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714048 00000100:00100000:1.0:1767321014.162173:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915713920 00000100:00100000:1.0:1767321014.162177:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915713920 00000100:00100000:1.0:1767321014.162200:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2060 00000100:00100000:1.0:1767321014.162210:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915714048:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321014.162301:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915714048:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 92us (385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321014.162309:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2060 00000100:00100000:1.0:1767321014.162327:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2728 00000100:00100000:1.0:1767321014.162331:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713920:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321014.162350:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321014.162372:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915713920:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (383us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321014.162379:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2728 00000100:00080000:0.0:1767321014.162460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321014.407601:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321015.428476:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321015.748580:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983215744:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321015.748664:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321015.748808:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215744 00000100:00100000:1.0:1767321015.748817:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215744 00000100:00100000:1.0:1767321015.748883:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1374 00000100:00100000:1.0:1767321015.748892:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983215744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321015.748921:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321015.748979:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983215744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 88us (315us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321015.748987:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1374 00000100:00100000:1.0:1767321015.749032:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463190ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983215744:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321015.749131:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321015.752907:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983215872:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321015.752978:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321015.753699:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983215872 00000100:00100000:1.0:1767321015.753709:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983215872 00000100:00100000:1.0:1767321015.753772:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1375 00000100:00100000:1.0:1767321015.753782:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983215872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321015.753815:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321015.753876:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983215872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (897us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321015.753884:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1375 00000100:00100000:3.0:1767321015.754236:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983215872:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321015.754827:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321016.452535:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321017.285711:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767320992/real 1767320992] req@ffff934631988e00 x1853163983210496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321017 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321017.285758:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934631988e00 x1853163983210496/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321017 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321017.285776:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321017.285801:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321017.285804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983210496:0@lo:250:kworker.0 00000100:00080000:3.0:1767321017.285936:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321017.285942:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321017.285945:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321017.285950:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6241 00000100:00080000:3.0:1767321017.285953:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321017.285960:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321017.285994:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375180 x1853163983216000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321017.286069:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321017.286072:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321017.286074:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321017.286078:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6261 00000100:00080000:3.0:1767321017.286080:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1767321017.286233:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983216128:0@lo:400:kworker.0 00000100:00100000:2.0:1767321017.286329:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321017.286415:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216128 00000100:00100000:2.0:1767321017.286423:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216128 00000100:00100000:2.0:1767321017.286483:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061 00000100:00100000:2.0:1767321017.286492:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983216128:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321017.286550:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983216128:12345-0@lo:400:kworker.0 Request processed in 59us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321017.286558:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061 00000100:00080000:2.0:1767321017.286581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767321017.286619:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375500 x1853163983216128/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321033 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321017.286631:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983216128:0@lo:400:kworker.0 00000100:00080000:3.0:1767321017.286938:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321017.286959:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604376680 x1853163983216256/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321017.286998:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321017.287001:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321017.287003:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321017.287007:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6261 00000100:00080000:3.0:1767321017.287009:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321017.287013:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321017.287031:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595c00 x1853163983216512/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321017.287038:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321017.287041:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321017.287043:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321017.287045:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6261 00000100:00080000:3.0:1767321017.287048:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321017.287051:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321017.287059:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638595500 x1853163983216640/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321017.287079:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321017.287082:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321017.287176:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321017.287180:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6261 00000100:00080000:3.0:1767321017.287182:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321017.287185:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321017.287306:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638594a80 x1853163983217024/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321017.287476:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983216768:0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.287521:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321017.287555:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321017.287561:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983216384:0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.287584:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321017.287623:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216384 00000100:00100000:0.0:1767321017.287628:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216384 00000100:00100000:0.0:1767321017.287674:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2062 00000100:00100000:0.0:1767321017.287681:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983216384:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.287722:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983216384:12345-0@lo:400:kworker.0 Request processed in 41us (137us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321017.287728:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2062 00000100:00100000:0.0:1767321017.287748:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983216896:0@lo:400:kworker.0 00000100:00100000:0.0:1767321017.287769:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321017.287793:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767321017.287813:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604375500 x1853163983216384/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321033 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321017.287822:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983216384:0@lo:400:kworker.0 00000100:00100000:1.0:1767321017.287980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983216000:0@lo:250:kworker.0 00000400:00000100:1.0:1767321017.288005:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983216000 offset 0 length 520: 4 00000100:00100000:1.0:1767321017.288038:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983216256:0@lo:38:kworker.0 00000100:00100000:1.0:1767321017.288059:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.288095:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983216512:0@lo:38:kworker.0 00000100:00100000:1.0:1767321017.288113:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.288128:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983216640:0@lo:38:kworker.0 00000100:00100000:1.0:1767321017.288148:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.288164:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983217024:0@lo:38:kworker.0 00000100:00100000:1.0:1767321017.288180:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321017.288497:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216256 00000100:00100000:1.0:1767321017.288503:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216256 00000100:00100000:1.0:1767321017.288524:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216512 00000100:00100000:1.0:1767321017.288526:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216512 00000100:00100000:1.0:1767321017.288534:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217024 00000100:00100000:1.0:1767321017.288536:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217024 00000100:00100000:1.0:1767321017.288545:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2729 00000100:00100000:1.0:1767321017.288550:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983216256:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321017.288567:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321017.288611:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983216256:12345-0@lo:38:kworker.0 Request processed in 50us (541us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321017.288618:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2729 00000100:00100000:1.0:1767321017.288631:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2730 00000100:00100000:1.0:1767321017.288635:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983216512:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321017.288643:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321017.288666:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983216512:12345-0@lo:38:kworker.0 Request processed in 29us (552us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321017.288670:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2730 00000100:00100000:1.0:1767321017.288679:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2731 00000100:00100000:1.0:1767321017.288683:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983217024:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321017.288691:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321017.288711:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983217024:12345-0@lo:38:kworker.0 Request processed in 27us (531us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321017.288717:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2731 00000100:00100000:1.0:1767321017.288733:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216640 00000100:00100000:1.0:1767321017.288736:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216640 00000100:00100000:1.0:1767321017.288753:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 707 00000100:00100000:1.0:1767321017.288757:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983216640:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321017.288766:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321017.288790:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983216640:12345-0@lo:38:kworker.0 Request processed in 32us (641us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321017.288795:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 707 00000100:00080000:1.0:1767321017.288831:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321017.288837:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321017.288840:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983216256:0@lo:38:kworker.0 00000100:00080000:1.0:1767321017.288848:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321017.288851:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321017.288854:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983216512:0@lo:38:kworker.0 00000100:00080000:1.0:1767321017.288861:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321017.288864:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321017.288867:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638595500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983216640:0@lo:38:kworker.0 00000100:00080000:1.0:1767321017.288874:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321017.288877:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321017.288879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983217024:0@lo:38:kworker.0 00000100:00100000:3.0:1767321017.289011:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216768 00000100:00100000:3.0:1767321017.289016:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216768 00000100:00100000:3.0:1767321017.289077:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983216896 00000100:00100000:3.0:1767321017.289081:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983216896 00000100:00100000:3.0:1767321017.289102:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1263 00000100:00100000:3.0:1767321017.289109:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983216768:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321017.289152:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983216768:12345-0@lo:400:kworker.0 Request processed in 43us (1630us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321017.289157:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1263 00000100:00100000:3.0:1767321017.289172:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1264 00000100:00100000:3.0:1767321017.289178:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983216896:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321017.289368:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321017.289372:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321017.289398:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983216768:0@lo:400:kworker.0 00000100:00100000:3.0:1767321017.289491:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983216896:12345-0@lo:400:kworker.0 Request processed in 312us (1719us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321017.289498:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1264 00000100:00100000:0.0:1767321017.289566:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638596300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983216896:0@lo:400:kworker.0 00010000:00010000:3.0:1767321017.477576:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321018.501175:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321019.278445:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321019.279077:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714304 00000100:00100000:2.0:1767321019.279089:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714304 00000100:00080000:0.0:1767321019.279189:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321019.279319:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2063 00000100:00100000:2.0:1767321019.279332:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915714304:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321019.279431:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915714304:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 102us (996us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321019.279439:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2063 00000100:00100000:3.0:1767321019.280742:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321019.291894:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714176 00000100:00100000:1.0:1767321019.291910:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714176 00000100:00100000:1.0:1767321019.291984:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2732 00000100:00100000:1.0:1767321019.291993:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915714176:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321019.292016:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321019.292094:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915714176:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 101us (11359us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321019.292102:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2732 00010000:00010000:2.0:1767321019.524480:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321020.551121:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321020.869522:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983217152:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321020.869581:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321020.869596:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983217280:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321020.869604:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321020.869643:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217152 00000100:00100000:3.0:1767321020.869652:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217152 00000100:00100000:3.0:1767321020.869725:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1376 00000100:00100000:3.0:1767321020.869736:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983217152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321020.869780:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767321020.869842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321020.875658:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321020.875907:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217280 00000100:00100000:1.0:1767321020.875919:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217280 00000100:00100000:1.0:1767321020.875996:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1377 00000100:00100000:1.0:1767321020.876007:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983217280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321020.876047:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321020.876109:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983217280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (458us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321020.876117:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1377 00000100:00080000:0.0:1767321020.877410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321020.877473:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631909c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983217280:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321020.881883:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983217152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 12137us (12266us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321020.881901:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1376 00000100:00100000:2.0:1767321020.881990:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983217152:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321021.572613:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321022.407498:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321022.407523:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767321022.407624:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983217408:0@lo:400:kworker.0 00000100:00100000:0.0:1767321022.407706:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.407813:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217408 00000100:00100000:2.0:1767321022.407822:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217408 00000100:00100000:2.0:1767321022.407891:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064 00000100:00100000:2.0:1767321022.407902:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983217408:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321022.407992:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321022.408321:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321022.408326:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321022.408330:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321022.408336:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6265 00000100:00080000:1.0:1767321022.408340:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321022.408346:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321022.408375:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059cca80 x1853163983217536/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321022.408413:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321022.408416:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321022.408418:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321022.408421:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6265 00000100:00080000:1.0:1767321022.408423:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321022.408427:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321022.408439:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983217792/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321022.408447:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321022.408451:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321022.408453:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321022.408455:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6265 00000100:00080000:1.0:1767321022.408458:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321022.408461:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321022.408472:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163983217920/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321022.408547:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983218176:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408591:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.408634:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983217664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408658:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.408676:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321022.408682:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983218048:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408703:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.408731:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218176 00000100:00100000:3.0:1767321022.408736:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218176 00000100:00100000:3.0:1767321022.408772:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218048 00000100:00100000:3.0:1767321022.408776:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218048 00000100:00100000:3.0:1767321022.408797:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1265 00000100:00100000:3.0:1767321022.408804:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983218176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408847:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983218176:12345-0@lo:400:kworker.0 Request processed in 43us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321022.408855:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1265 00000100:00100000:3.0:1767321022.408872:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1266 00000100:00100000:3.0:1767321022.408877:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983218048:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408907:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983218048:12345-0@lo:400:kworker.0 Request processed in 30us (203us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321022.408913:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1266 00000100:00100000:3.0:1767321022.408936:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217664 00000100:00100000:3.0:1767321022.408940:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217664 00000100:00100000:3.0:1767321022.408964:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2065 00000100:00100000:3.0:1767321022.408970:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983217664:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.408996:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638595500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983217664:12345-0@lo:400:kworker.0 Request processed in 27us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321022.409001:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2065 00000100:00080000:3.0:1767321022.409019:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321022.409022:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321022.409024:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321022.409051:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983218048:0@lo:400:kworker.0 00010000:00080000:3.0:1767321022.409062:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076faa00 x1853163983217664/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321038 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321022.409071:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983217664:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.409098:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983218176:0@lo:400:kworker.0 00000100:00100000:3.0:1767321022.409153:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983217536:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.409178:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.409195:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983217792:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.409211:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.409224:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983217920:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.409243:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321022.409268:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217536 00000100:00100000:3.0:1767321022.409294:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217536 00000100:00100000:3.0:1767321022.409314:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217792 00000100:00100000:3.0:1767321022.409317:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217792 00000100:00100000:3.0:1767321022.409330:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2733 00000100:00100000:3.0:1767321022.409334:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983217536:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321022.409353:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321022.409385:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983217536:12345-0@lo:38:kworker.0 Request processed in 49us (206us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321022.409391:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2733 00000100:00100000:3.0:1767321022.409402:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2734 00000100:00100000:3.0:1767321022.409405:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983217792:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321022.409414:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321022.409435:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631902a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983217792:12345-0@lo:38:kworker.0 Request processed in 28us (222us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321022.409441:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2734 00000100:00080000:3.0:1767321022.409464:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321022.409469:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321022.409472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983217536:0@lo:38:kworker.0 00000100:00080000:3.0:1767321022.409480:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321022.409482:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321022.409483:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983217792:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.409525:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983217920 00000100:00100000:3.0:1767321022.409529:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983217920 00000100:00100000:3.0:1767321022.409549:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 708 00000100:00100000:3.0:1767321022.409554:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983217920:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321022.409562:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321022.409591:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983217920:12345-0@lo:38:kworker.0 Request processed in 36us (347us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321022.409597:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 708 00000100:00080000:3.0:1767321022.409618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321022.409622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321022.409625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983217920:0@lo:38:kworker.0 00000100:00100000:2.0:1767321022.411485:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983217408:12345-0@lo:400:kworker.0 Request processed in 3580us (3772us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321022.411497:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064 00010000:00080000:0.0:1767321022.411606:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059ced80 x1853163983217408/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321038 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321022.411621:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ced80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983217408:0@lo:400:kworker.0 00000100:00080000:1.0:1767321022.412527:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321022.412535:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321022.412540:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321022.412546:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6265 00000100:00080000:1.0:1767321022.412550:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321022.412556:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321022.412596:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163983218304/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321022.412714:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983218304:0@lo:38:kworker.0 00000100:00100000:3.0:1767321022.412774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321022.412923:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218304 00000100:00100000:2.0:1767321022.412930:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218304 00000100:00100000:2.0:1767321022.412971:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2735 00000100:00100000:2.0:1767321022.412978:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983218304:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321022.412999:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321022.413358:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983218304:12345-0@lo:38:kworker.0 Request processed in 377us (581us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321022.413366:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2735 00000100:00080000:0.0:1767321022.418405:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321022.418422:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321022.418429:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983218304:0@lo:38:kworker.0 00010000:00010000:2.0:1767321022.600494:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321023.620612:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321024.400726:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321024.400817:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321024.401086:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714560 00000100:00100000:3.0:1767321024.401105:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714560 00000100:00100000:3.0:1767321024.401183:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714432 00000100:00100000:3.0:1767321024.401186:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714432 00000100:00100000:3.0:1767321024.401211:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2066 00000100:00100000:3.0:1767321024.401223:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915714560:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767321024.401394:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321024.401431:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2736 00000100:00100000:0.0:1767321024.401447:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915714432:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321024.401500:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321024.401558:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915714560:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 337us (840us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321024.401565:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2066 00000100:00100000:0.0:1767321024.405308:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915714432:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 3848us (4478us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321024.405329:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2736 00010000:00010000:3.0:1767321024.644565:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321025.668520:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321025.988575:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321025.988600:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983218432:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321025.988671:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321025.988891:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218432 00000100:00100000:1.0:1767321025.988900:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218432 00000100:00100000:1.0:1767321025.988971:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1378 00000100:00100000:1.0:1767321025.988980:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983218432:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321025.989010:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321025.989056:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983218432:12345-0@lo:13:osp-pre-0-1.0 Request processed in 77us (385us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321025.989064:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1378 00000100:00080000:0.0:1767321025.989123:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321025.989236:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983218432:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321025.989733:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321025.989743:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983218560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321025.989784:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321025.989892:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218560 00000100:00100000:1.0:1767321025.989897:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218560 00000100:00100000:1.0:1767321025.989940:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1379 00000100:00100000:1.0:1767321025.989948:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346324c0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983218560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321025.989966:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321025.990009:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346324c0c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983218560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (223us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321025.990015:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1379 00000100:00080000:0.0:1767321025.990301:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321025.990869:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983218560:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321026.693870:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321027.530959:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321027.530974:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321027.531076:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321027.531081:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321027.531088:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321027.531099:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6270 00000100:00080000:2.0:1767321027.531106:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321027.531116:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321027.531142:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463198b480 x1853163983218816/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321027.531211:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321027.531215:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321027.531217:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321027.531221:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6270 00000100:00080000:2.0:1767321027.531224:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321027.531228:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321027.531243:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e247800 x1853163983219072/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321027.531252:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321027.531255:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321027.531257:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321027.531261:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6270 00000100:00080000:2.0:1767321027.531264:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321027.531268:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321027.531330:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246d80 x1853163983219200/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321027.531365:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321027.531370:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321027.531372:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321027.531376:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6270 00000100:00080000:2.0:1767321027.531378:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321027.531382:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321027.531394:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246300 x1853163983219584/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321027.531447:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983219328:0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.531538:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321027.531591:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321027.531598:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983218944:0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.531633:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321027.531797:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218944 00000100:00100000:2.0:1767321027.531807:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218944 00000100:00100000:2.0:1767321027.531877:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2067 00000100:00100000:2.0:1767321027.531889:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983218944:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321027.531953:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983218944:12345-0@lo:400:kworker.0 Request processed in 67us (319us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321027.531961:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2067 00010000:00080000:2.0:1767321027.532008:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631988700 x1853163983218944/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321043 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321027.532025:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631988700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983218944:0@lo:400:kworker.0 00000100:00100000:1.0:1767321027.532311:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983218816:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.532364:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.532394:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983219072:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.532427:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.532447:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983219200:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.532474:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.532516:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983219584:0@lo:38:kworker.0 00000100:00100000:1.0:1767321027.532536:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321027.532571:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218816 00000100:00100000:1.0:1767321027.532576:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218816 00000100:00100000:1.0:1767321027.532598:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219072 00000100:00100000:1.0:1767321027.532600:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219072 00000100:00100000:1.0:1767321027.532612:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219584 00000100:00100000:1.0:1767321027.532614:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219584 00000100:00100000:1.0:1767321027.532624:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2737 00000100:00100000:1.0:1767321027.532630:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983218816:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.532665:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.532706:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983218816:12345-0@lo:38:kworker.0 Request processed in 75us (342us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.532714:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2737 00000100:00100000:1.0:1767321027.532729:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2738 00000100:00100000:1.0:1767321027.532733:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983219072:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.532742:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.532767:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983219072:12345-0@lo:38:kworker.0 Request processed in 33us (340us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.532773:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2738 00000100:00100000:1.0:1767321027.532783:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2739 00000100:00100000:1.0:1767321027.532786:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983219584:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321027.532795:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321027.532816:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983219584:12345-0@lo:38:kworker.0 Request processed in 28us (279us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321027.532822:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2739 00000100:00080000:1.0:1767321027.532856:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321027.532861:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321027.532865:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983218816:0@lo:38:kworker.0 00000100:00080000:1.0:1767321027.532874:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321027.532877:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321027.532880:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983219072:0@lo:38:kworker.0 00000100:00080000:1.0:1767321027.532888:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321027.532892:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321027.532895:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983219584:0@lo:38:kworker.0 00000100:00080000:0.0:1767321027.533581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321027.533609:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983218688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321027.533673:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321027.533726:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321027.533733:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983219456:0@lo:400:kworker.0 00000100:00100000:0.0:1767321027.533761:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321027.533949:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219200 00000100:00100000:3.0:1767321027.533958:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219200 00000100:00100000:3.0:1767321027.534006:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 709 00000100:00100000:3.0:1767321027.534012:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983219200:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321027.534029:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321027.534090:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983219200:12345-0@lo:38:kworker.0 Request processed in 77us (1615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321027.534097:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 709 00000100:00100000:3.0:1767321027.534123:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219328 00000100:00100000:3.0:1767321027.534127:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219328 00000100:00100000:3.0:1767321027.534178:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219456 00000100:00100000:3.0:1767321027.534182:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219456 00000100:00100000:3.0:1767321027.534205:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1267 00000100:00100000:3.0:1767321027.534212:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983219328:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321027.534268:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983219328:12345-0@lo:400:kworker.0 Request processed in 56us (2729us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321027.534324:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1267 00000100:00100000:3.0:1767321027.534337:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1268 00000100:00100000:3.0:1767321027.534357:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983219456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321027.534389:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983219456:12345-0@lo:400:kworker.0 Request processed in 46us (627us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321027.534396:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1268 00000100:00100000:3.0:1767321027.534452:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983218688 00000100:00100000:3.0:1767321027.534456:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983218688 00000100:00100000:3.0:1767321027.534483:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2068 00000100:00100000:3.0:1767321027.534488:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983218688:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321027.534521:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983218688:12345-0@lo:400:kworker.0 Request processed in 33us (848us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321027.534527:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2068 00000100:00100000:3.0:1767321027.534575:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983219328:0@lo:400:kworker.0 00000100:00080000:3.0:1767321027.534618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321027.534623:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321027.534627:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983219200:0@lo:38:kworker.0 00000100:00080000:0.0:1767321027.534993:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321027.534999:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321027.535002:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321027.535038:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983219456:0@lo:400:kworker.0 00010000:00080000:0.0:1767321027.535053:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463198b800 x1853163983218688/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321043 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321027.535076:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463198b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983218688:0@lo:400:kworker.0 00010000:00010000:3.0:1767321027.718319:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321028.741903:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321029.518088:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321029.518230:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321029.523974:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714816 00000100:00100000:2.0:1767321029.523991:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714816 00000100:00100000:2.0:1767321029.524128:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714688 00000100:00100000:2.0:1767321029.524133:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714688 00000100:00100000:2.0:1767321029.524160:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2069 00000100:00100000:2.0:1767321029.524170:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915714816:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321029.524251:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915714816:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 82us (6173us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321029.524258:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2069 00000100:00100000:2.0:1767321029.524293:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2740 00000100:00100000:2.0:1767321029.524298:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915714688:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321029.524329:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321029.524358:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915714688:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 58us (6129us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321029.524365:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2740 00000100:00080000:0.0:1767321029.525765:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321029.764551:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321030.792686:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321031.108612:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321031.108641:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983219840:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321031.108715:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321031.108899:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219840 00000100:00100000:0.0:1767321031.108908:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219840 00000100:00100000:2.0:1767321031.108941:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321031.108955:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983219712:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321031.108983:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1380 00000100:00100000:2.0:1767321031.108990:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321031.108997:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983219840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321031.109032:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321031.109094:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983219840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 102us (378us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321031.109103:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1380 00000100:00100000:0.0:1767321031.109119:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219712 00000100:00100000:2.0:1767321031.109181:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346392c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983219840:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321031.109400:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219712 00000100:00100000:0.0:1767321031.109427:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1381 00000100:00100000:0.0:1767321031.109434:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983219712:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321031.109451:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321031.109497:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983219712:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (508us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321031.109504:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1381 00000100:00080000:1.0:1767321031.109614:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321031.109619:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321031.109663:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983219712:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321031.812609:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321032.644426:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321032.644438:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321032.644509:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321032.644514:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321032.644517:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321032.644526:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6276 00000100:00080000:3.0:1767321032.644531:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321032.644538:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321032.644561:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901180 x1853163983220096/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321032.644699:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983219968:0@lo:400:kworker.0 00000100:00100000:2.0:1767321032.644774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.644851:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983219968 00000100:00100000:2.0:1767321032.644858:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983219968 00000100:00100000:2.0:1767321032.644906:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070 00000100:00100000:2.0:1767321032.644915:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983219968:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321032.644964:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983219968:12345-0@lo:400:kworker.0 Request processed in 50us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321032.644972:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070 00010000:00080000:2.0:1767321032.645009:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631900a80 x1853163983219968/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321048 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321032.645022:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983219968:0@lo:400:kworker.0 00000100:00080000:3.0:1767321032.645401:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321032.645405:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321032.645408:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321032.645412:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6276 00000100:00080000:3.0:1767321032.645415:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321032.645421:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321032.645436:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901c00 x1853163983220352/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321032.645444:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321032.645447:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321032.645449:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321032.645452:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6276 00000100:00080000:3.0:1767321032.645454:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321032.645457:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321032.645467:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631900a80 x1853163983220480/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321032.645492:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321032.645496:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321032.645498:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321032.645501:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6276 00000100:00080000:3.0:1767321032.645504:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321032.645507:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321032.645516:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059ce300 x1853163983220864/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321032.645546:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983220736:0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.645679:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983220224:0@lo:400:kworker.0 00000100:00080000:1.0:1767321032.645750:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321032.645904:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983220096:0@lo:38:kworker.0 00000100:00100000:0.0:1767321032.646148:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321032.646185:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321032.646192:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983220608:0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.646221:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321032.647825:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220608 00000100:00100000:1.0:1767321032.647841:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220608 00000100:00100000:1.0:1767321032.647912:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1269 00000100:00100000:1.0:1767321032.647922:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983220608:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321032.648004:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983220608:12345-0@lo:400:kworker.0 Request processed in 83us (1782us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321032.648012:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1269 00000100:00100000:1.0:1767321032.648037:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220224 00000100:00100000:1.0:1767321032.648041:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220224 00000100:00100000:1.0:1767321032.648075:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2071 00000100:00100000:1.0:1767321032.648082:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983220224:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321032.648128:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983220224:12345-0@lo:400:kworker.0 Request processed in 47us (1980us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321032.648135:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2071 00000100:00080000:1.0:1767321032.648151:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321032.648193:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983220608:0@lo:400:kworker.0 00010000:00080000:1.0:1767321032.648208:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934631903800 x1853163983220224/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321048 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321032.648219:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631903800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983220224:0@lo:400:kworker.0 00000100:00100000:3.0:1767321032.650198:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321032.650341:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220736 00000100:00100000:0.0:1767321032.650349:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220736 00000100:00100000:0.0:1767321032.650396:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1270 00000100:00100000:0.0:1767321032.650404:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983220736:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321032.650473:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ced80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983220736:12345-0@lo:400:kworker.0 Request processed in 70us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321032.650481:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1270 00000100:00100000:3.0:1767321032.651311:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983220736:0@lo:400:kworker.0 00000100:00100000:2.0:1767321032.651721:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.651768:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983220352:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.651797:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.651824:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983220480:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.651845:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.651870:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983220864:0@lo:38:kworker.0 00000100:00100000:2.0:1767321032.651888:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321032.651919:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220096 00000100:00100000:2.0:1767321032.651924:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220096 00000100:00100000:2.0:1767321032.651944:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220352 00000100:00100000:2.0:1767321032.651946:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220352 00000100:00100000:2.0:1767321032.651957:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220864 00000100:00100000:2.0:1767321032.651959:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220864 00000100:00100000:2.0:1767321032.651968:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2741 00000100:00100000:2.0:1767321032.651974:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220096:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321032.651993:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321032.652026:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220096:12345-0@lo:38:kworker.0 Request processed in 51us (306us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321032.652032:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2741 00000100:00100000:2.0:1767321032.652056:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2742 00000100:00100000:2.0:1767321032.652060:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220352:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321032.652069:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321032.652087:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463198b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220352:12345-0@lo:38:kworker.0 Request processed in 25us (290us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321032.652092:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2742 00000100:00100000:2.0:1767321032.652101:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2743 00000100:00100000:2.0:1767321032.652104:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321032.652112:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321032.652134:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983220864:12345-0@lo:38:kworker.0 Request processed in 28us (245us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321032.652139:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2743 00000100:00080000:2.0:1767321032.652165:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.652171:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321032.652174:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983220096:0@lo:38:kworker.0 00000100:00080000:2.0:1767321032.652181:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.652184:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321032.652187:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983220352:0@lo:38:kworker.0 00000100:00080000:2.0:1767321032.652193:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.652195:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321032.652198:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059ce300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983220864:0@lo:38:kworker.0 00000100:00100000:0.0:1767321032.652482:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220480 00000100:00100000:0.0:1767321032.652487:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220480 00000100:00100000:0.0:1767321032.652518:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 710 00000100:00100000:0.0:1767321032.652522:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983220480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321032.652533:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321032.652568:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631988700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983220480:12345-0@lo:38:kworker.0 Request processed in 45us (722us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321032.652582:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 710 00000100:00080000:2.0:1767321032.652825:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321032.652830:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321032.652833:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983220480:0@lo:38:kworker.0 00010000:00010000:3.0:1767321032.836998:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321033.861114:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321034.635694:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321034.635800:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715072 00000100:00100000:1.0:1767321034.635811:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715072 00000100:00100000:1.0:1767321034.635899:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2072 00000100:00100000:1.0:1767321034.635909:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915715072:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321034.635991:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a580000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915715072:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321034.635999:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2072 00000100:00080000:2.0:1767321034.636220:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321034.636424:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321034.636733:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915714944 00000100:00100000:0.0:1767321034.636740:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915714944 00000100:00100000:0.0:1767321034.636776:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2744 00000100:00100000:0.0:1767321034.636782:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915714944:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321034.636802:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321034.636849:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915714944:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 66us (424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321034.636857:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2744 00010000:00010000:2.0:1767321034.884479:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321035.909132:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321036.229808:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983221120:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321036.229823:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321036.229839:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983220992:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321036.229881:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321036.229912:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221120 00000100:00100000:2.0:1767321036.229920:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221120 00000100:00100000:3.0:1767321036.229967:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321036.229977:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983220992 00000100:00100000:2.0:1767321036.229982:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983220992 00000100:00080000:3.0:1767321036.230003:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321036.230007:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1382 00000100:00080000:3.0:1767321036.230008:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321036.230021:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983221120:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321036.230026:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1383 00000100:00100000:3.0:1767321036.230034:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983220992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321036.230053:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00002000:00000024:2.0:1767321036.230054:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321036.230093:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059ce300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983220992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (127us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321036.230099:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1383 00000100:00100000:2.0:1767321036.230110:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983221120:12345-0@lo:13:osp-pre-0-1.0 Request processed in 91us (228us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321036.230119:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1382 00000100:00100000:3.0:1767321036.230150:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983220992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321036.230158:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059cdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983221120:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767321036.933088:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321037.764401:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321037.764413:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321037.764480:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767321037.764486:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983221248:0@lo:400:kworker.0 00000100:00080000:2.0:1767321037.764487:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321037.764491:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321037.764498:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321037.764502:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321037.764509:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321037.764535:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395ea300 x1853163983221376/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321037.764558:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321037.764571:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321037.764576:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321037.764578:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321037.764583:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321037.764586:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321037.764590:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321037.764600:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983221376:0@lo:38:kworker.0 00000100:00100000:2.0:1767321037.764605:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e9f80 x1853163983221632/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321037.764617:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321037.764620:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321037.764622:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321037.764626:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321037.764628:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321037.764633:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321037.764640:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321037.764647:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346395e8a80 x1853163983221760/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321037.764676:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221248 00000100:00080000:2.0:1767321037.764677:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321037.764682:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321037.764684:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767321037.764686:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221248 00000100:00080000:2.0:1767321037.764688:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6281 00000100:00080000:2.0:1767321037.764693:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321037.764696:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321037.764709:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ea00 x1853163983222144/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321037.764736:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983221504:0@lo:400:kworker.0 00000100:00100000:0.0:1767321037.764748:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221376 00000100:00100000:3.0:1767321037.764749:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321037.764753:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221376 00000100:00100000:3.0:1767321037.764757:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983222016:0@lo:400:kworker.0 00000100:00100000:0.0:1767321037.764774:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073 00000100:00100000:2.0:1767321037.764780:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321037.764789:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983221248:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321037.764794:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321037.764811:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321037.764821:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983221888:0@lo:400:kworker.0 00000100:00080000:3.0:1767321037.764824:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321037.764844:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983221632:0@lo:38:kworker.0 00000100:00100000:0.0:1767321037.764864:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059cdc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983221248:12345-0@lo:400:kworker.0 Request processed in 78us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321037.764868:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321037.764875:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321037.764881:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073 00000100:00100000:3.0:1767321037.764885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983221760:0@lo:38:kworker.0 00000100:00100000:0.0:1767321037.764905:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221504 00000100:00100000:0.0:1767321037.764910:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221504 00000100:00100000:3.0:1767321037.764915:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321037.764922:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221632 00000100:00100000:2.0:1767321037.764928:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221632 00000100:00100000:3.0:1767321037.764951:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983222144:0@lo:38:kworker.0 00000100:00100000:2.0:1767321037.764953:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2745 00000100:00100000:2.0:1767321037.764962:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983221376:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321037.764975:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:2.0:1767321037.764995:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321037.765013:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221760 00000100:00100000:3.0:1767321037.765018:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221760 00000100:00100000:2.0:1767321037.765039:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a581500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983221376:12345-0@lo:38:kworker.0 Request processed in 74us (398us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321037.765046:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 711 00000100:00100000:2.0:1767321037.765051:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2745 00000100:00100000:3.0:1767321037.765056:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983221760:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321037.765066:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222144 00010000:00020000:3.0:1767321037.765068:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321037.765073:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222144 00000100:00100000:2.0:1767321037.765092:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2746 00000100:00100000:2.0:1767321037.765097:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983221632:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321037.765103:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983221760:12345-0@lo:38:kworker.0 Request processed in 46us (186us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767321037.765111:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321037.765113:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 711 00000100:00100000:2.0:1767321037.765150:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983221632:12345-0@lo:38:kworker.0 Request processed in 50us (282us total) trans 0 rc -19/-19 00010000:00080000:3.0:1767321037.765157:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ead80 x1853163983221248/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321053 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321037.765158:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2746 00000100:00100000:3.0:1767321037.765174:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983221248:0@lo:400:kworker.0 00000100:00100000:2.0:1767321037.765176:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2747 00000100:00100000:2.0:1767321037.765184:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983222144:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321037.765194:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321037.765232:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983222144:12345-0@lo:38:kworker.0 Request processed in 35us (245us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321037.765237:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2074 00000100:00100000:3.0:1767321037.765248:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983221504:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321037.765304:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222016 00000100:00100000:1.0:1767321037.765309:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222016 00000100:00100000:3.0:1767321037.765314:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983221504:12345-0@lo:400:kworker.0 Request processed in 68us (537us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321037.765346:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983221888 00000100:00100000:1.0:1767321037.765349:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983221888 00000100:00100000:1.0:1767321037.765369:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1271 00000100:00100000:1.0:1767321037.765375:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983222016:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321037.765415:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983222016:12345-0@lo:400:kworker.0 Request processed in 40us (624us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321037.765431:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1271 00000100:00100000:1.0:1767321037.765448:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1272 00000100:00100000:1.0:1767321037.765459:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983221888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321037.765484:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604323480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983221888:12345-0@lo:400:kworker.0 Request processed in 31us (615us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321037.765490:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1272 00000100:00080000:1.0:1767321037.765542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321037.765571:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395eb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983221888:0@lo:400:kworker.0 00010000:00080000:1.0:1767321037.765581:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346395ea680 x1853163983221504/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321053 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321037.765590:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983221504:0@lo:400:kworker.0 00000100:00080000:1.0:1767321037.765615:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321037.765620:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321037.765623:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983221632:0@lo:38:kworker.0 00000100:00080000:1.0:1767321037.765631:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321037.765634:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321037.765637:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983221760:0@lo:38:kworker.0 00000100:00080000:1.0:1767321037.765644:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321037.765647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321037.765649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983222144:0@lo:38:kworker.0 00000100:00080000:1.0:1767321037.765658:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321037.765660:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321037.765663:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395ea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983221376:0@lo:38:kworker.0 00000100:00100000:1.0:1767321037.766411:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983222016:0@lo:400:kworker.0 00000100:00100000:2.0:1767321037.772471:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2747 00000100:00100000:3.0:1767321037.777720:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2074 00010000:00010000:2.0:1767321037.956544:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321038.735655:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321038.736242:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715456 00000100:00100000:3.0:1767321038.736254:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715456 00000100:00100000:3.0:1767321038.736411:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2075 00000100:00100000:3.0:1767321038.736423:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715456:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321038.736516:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715456:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 94us (1252us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321038.736525:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2075 00000100:00080000:1.0:1767321038.737261:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:3.0:1767321038.738185:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915715200 offset 0 length 520: 4 00000100:00100000:3.0:1767321038.738241:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321038.738690:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715328 00000100:00100000:2.0:1767321038.738701:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715328 00000100:00100000:2.0:1767321038.738748:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2748 00000100:00100000:2.0:1767321038.738755:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915715328:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321038.738777:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321038.738847:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915715328:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (605us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321038.738854:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2748 00010000:00010000:3.0:1767321038.980693:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321040.004559:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321041.031804:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321041.348690:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983222272:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321041.348690:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321041.348706:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983222400:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321041.357490:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321041.357980:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222272 00000100:00100000:3.0:1767321041.357991:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222272 00000100:00100000:3.0:1767321041.358065:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1384 00000100:00100000:3.0:1767321041.358075:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983222272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321041.358108:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321041.358177:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983222272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 104us (695us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321041.358184:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1384 00000100:00100000:3.0:1767321041.358234:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983222272:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321041.358612:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321041.359565:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321041.359762:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222400 00000100:00100000:0.0:1767321041.359772:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222400 00000100:00080000:1.0:1767321041.359836:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321041.359971:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1385 00000100:00100000:0.0:1767321041.359982:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983222400:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321041.360013:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321041.360077:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983222400:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (516us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321041.360084:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1385 00000100:00100000:2.0:1767321041.360178:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983222400:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321042.052780:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321042.884465:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321017/real 1767321017] req@ffff934604375180 x1853163983216000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321042.884502:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934604375180 x1853163983216000/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321042.884524:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321042.884553:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321042.884558:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983216000:0@lo:250:kworker.0 00000100:00080000:1.0:1767321042.884651:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321042.884656:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321042.884660:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321042.884665:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6265 00000100:00080000:1.0:1767321042.884669:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321042.884725:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321042.884856:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e246300 x1853163983222528/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321042.884917:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983222528:0@lo:250:kworker.0 00000400:00000100:3.0:1767321042.884964:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983222528 offset 0 length 520: 4 00000100:00080000:1.0:1767321042.885059:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321042.885063:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321042.885065:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321042.885068:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6286 00000100:00080000:1.0:1767321042.885071:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321042.885075:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321042.885090:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93462e244e00 x1853163983222784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321042.885151:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321042.885163:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983222656:0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.885203:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321042.885530:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222656 00000100:00100000:2.0:1767321042.885539:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222656 00000100:00100000:2.0:1767321042.885603:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076 00000100:00100000:2.0:1767321042.885614:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983222656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321042.885672:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983222656:12345-0@lo:400:kworker.0 Request processed in 60us (468us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321042.885681:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076 00010000:00080000:2.0:1767321042.885724:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e247800 x1853163983222656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321058 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321042.885737:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983222656:0@lo:400:kworker.0 00000100:00080000:0.0:1767321042.886029:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321042.888910:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983222784:0@lo:38:kworker.0 00000100:00100000:3.0:1767321042.888985:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321042.893977:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321042.893984:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321042.893989:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321042.893997:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6286 00000100:00080000:1.0:1767321042.894002:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321042.894008:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321042.894031:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543dc00 x1853163983223040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321042.894043:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321042.894046:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321042.894048:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321042.894052:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6286 00000100:00080000:1.0:1767321042.894054:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321042.894058:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321042.894070:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460543ea00 x1853163983223168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321042.894103:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321042.894106:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321042.894107:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321042.894111:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6286 00000100:00080000:1.0:1767321042.894113:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321042.894117:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321042.894138:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934631901180 x1853163983223552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321042.894175:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321042.894196:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983223424:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.894267:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321042.894334:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223424 00000100:00100000:1.0:1767321042.894341:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223424 00000100:00100000:1.0:1767321042.894388:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1273 00000100:00100000:1.0:1767321042.894396:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983223424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.894446:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631903800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983223424:12345-0@lo:400:kworker.0 Request processed in 50us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321042.894452:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1273 00000100:00100000:1.0:1767321042.894501:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983223424:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.894523:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983223296:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.894545:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321042.894565:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321042.894570:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983222912:0@lo:400:kworker.0 00000100:00100000:1.0:1767321042.894594:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321042.894683:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223296 00000100:00100000:0.0:1767321042.894688:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223296 00000100:00100000:0.0:1767321042.894717:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1274 00000100:00100000:0.0:1767321042.894724:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983223296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321042.894783:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983223296:12345-0@lo:400:kworker.0 Request processed in 59us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321042.894789:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1274 00000100:00100000:3.0:1767321042.894942:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983223040:0@lo:38:kworker.0 00000100:00100000:3.0:1767321042.903207:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321042.903333:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631900a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983223296:0@lo:400:kworker.0 00000100:00100000:2.0:1767321042.903669:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222784 00000100:00100000:2.0:1767321042.903678:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222784 00000100:00100000:2.0:1767321042.903729:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983222912 00000100:00100000:2.0:1767321042.903732:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983222912 00000100:00100000:2.0:1767321042.903790:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2077 00000100:00100000:2.0:1767321042.903799:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983222912:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321042.903881:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983222912:12345-0@lo:400:kworker.0 Request processed in 83us (9286us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321042.903889:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2077 00000100:00100000:2.0:1767321042.903908:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2749 00000100:00100000:2.0:1767321042.903912:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983222784:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321042.903934:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321042.903964:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983222784:12345-0@lo:38:kworker.0 Request processed in 51us (14980us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321042.903970:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2749 00000100:00080000:2.0:1767321042.903989:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321042.904268:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462e247800 x1853163983222912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321058 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321042.904293:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e247800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983222912:0@lo:400:kworker.0 00000100:00100000:3.0:1767321042.904535:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983223168:0@lo:38:kworker.0 00000100:00100000:3.0:1767321042.904582:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321042.904612:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983223552:0@lo:38:kworker.0 00000100:00100000:3.0:1767321042.904636:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321042.904662:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321042.904667:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321042.904671:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e244e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983222784:0@lo:38:kworker.0 00000100:00100000:3.0:1767321042.904699:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223040 00000100:00100000:3.0:1767321042.904703:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223040 00000100:00100000:3.0:1767321042.904724:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223552 00000100:00100000:3.0:1767321042.904727:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223552 00000100:00100000:3.0:1767321042.904744:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2750 00000100:00100000:3.0:1767321042.904750:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983223040:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321042.904778:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321042.904814:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f963480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983223040:12345-0@lo:38:kworker.0 Request processed in 63us (1618us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321042.904820:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2750 00000100:00100000:3.0:1767321042.904833:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2751 00000100:00100000:3.0:1767321042.904837:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983223552:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321042.904845:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321042.904867:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983223552:12345-0@lo:38:kworker.0 Request processed in 29us (231us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321042.904872:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2751 00000100:00080000:3.0:1767321042.904898:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321042.904903:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321042.904906:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983223040:0@lo:38:kworker.0 00000100:00080000:3.0:1767321042.904913:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321042.904916:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321042.904919:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934631901180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983223552:0@lo:38:kworker.0 00000100:00100000:2.0:1767321042.905209:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223168 00000100:00100000:2.0:1767321042.905214:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223168 00000100:00100000:2.0:1767321042.905244:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 712 00000100:00100000:2.0:1767321042.905249:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983223168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321042.905260:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321042.905324:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f961500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983223168:12345-0@lo:38:kworker.0 Request processed in 73us (741us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321042.905330:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 712 00000100:00080000:0.0:1767321042.905404:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321042.905409:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321042.905411:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460543ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983223168:0@lo:38:kworker.0 00010000:00010000:3.0:1767321043.077121:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321043.852398:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321043.852544:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715712 00000100:00100000:2.0:1767321043.852554:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715712 00000100:00100000:2.0:1767321043.852930:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2078 00000100:00100000:2.0:1767321043.852941:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715712:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321043.853134:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715712:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 195us (747us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321043.853143:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2078 00000100:00080000:0.0:1767321043.853207:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321043.856008:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321043.856103:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715584 00000100:00100000:2.0:1767321043.856127:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715584 00000100:00100000:2.0:1767321043.856179:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2752 00000100:00100000:2.0:1767321043.856188:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915715584:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321043.856211:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321043.856308:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631901180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915715584:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 118us (309us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321043.856315:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2752 00010000:00010000:2.0:1767321044.100507:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321045.124442:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321046.150446:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321046.468861:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983223680:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321046.468941:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321046.469247:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223680 00000100:00100000:0.0:1767321046.469255:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223680 00000100:00100000:0.0:1767321046.469343:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1386 00000100:00080000:1.0:1767321046.469350:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321046.469353:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983223680:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321046.469383:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321046.469438:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631900a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983223680:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (497us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321046.469465:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1386 00000100:00100000:1.0:1767321046.469477:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f962680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983223680:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321046.469620:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983223808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321046.469650:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321046.469967:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223808 00000100:00100000:3.0:1767321046.469973:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223808 00000100:00100000:3.0:1767321046.470020:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1387 00000100:00100000:3.0:1767321046.470027:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983223808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321046.470044:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321046.470089:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a34000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983223808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (438us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321046.470095:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1387 00000100:00100000:0.0:1767321046.470175:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463f960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983223808:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321046.470321:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321047.174300:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321048.004535:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321048.004545:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321048.004607:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321048.004611:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321048.004615:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321048.004622:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6291 00000100:00080000:0.0:1767321048.004627:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321048.004633:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321048.004660:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0a80 x1853163983224064/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321048.004667:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321048.004684:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983223936:0@lo:400:kworker.0 00000100:00080000:0.0:1767321048.004695:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321048.004700:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321048.004702:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321048.004705:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6291 00000100:00080000:0.0:1767321048.004708:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321048.004711:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321048.004725:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0700 x1853163983224320/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321048.004734:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321048.004738:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321048.004739:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321048.004742:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6291 00000100:00080000:0.0:1767321048.004744:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321048.004747:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321048.004757:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1f80 x1853163983224448/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321048.004778:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321048.004782:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321048.004783:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321048.004787:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6291 00000100:00080000:0.0:1767321048.004789:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321048.004793:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321048.004802:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b1180 x1853163983224832/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321048.004823:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983224192:0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.004872:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321048.004899:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321048.004966:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983224064:0@lo:38:kworker.0 00000100:00100000:1.0:1767321048.005009:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224192 00000100:00100000:1.0:1767321048.005017:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224192 00000100:00100000:1.0:1767321048.005080:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983223936 00000100:00100000:1.0:1767321048.005084:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983223936 00000100:00100000:1.0:1767321048.005107:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2079 00000100:00100000:1.0:1767321048.005116:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983224192:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.010099:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321048.010119:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983224576:0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.010191:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321048.010248:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2080 00000100:00100000:0.0:1767321048.010262:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983223936:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.010332:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983223936:12345-0@lo:400:kworker.0 Request processed in 75us (5433us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321048.010339:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2080 00000100:00100000:0.0:1767321048.010368:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224576 00000100:00100000:0.0:1767321048.010376:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224576 00000100:00100000:0.0:1767321048.010415:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1275 00000100:00100000:0.0:1767321048.010422:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983224576:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.010456:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983224576:12345-0@lo:400:kworker.0 Request processed in 35us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321048.010462:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1275 00000100:00100000:0.0:1767321048.010501:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983224576:0@lo:400:kworker.0 00000100:00080000:0.0:1767321048.010542:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321048.010547:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321048.011877:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321048.011895:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983224704:0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.011950:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321048.011990:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224704 00000100:00100000:0.0:1767321048.011996:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224704 00000100:00100000:0.0:1767321048.012045:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1276 00000100:00100000:0.0:1767321048.012054:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983224704:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321048.012099:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b3480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983224704:12345-0@lo:400:kworker.0 Request processed in 46us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321048.012105:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1276 00000100:00080000:0.0:1767321048.012123:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321048.012146:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983224704:0@lo:400:kworker.0 00010000:00080000:2.0:1767321048.014965:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609a35880 x1853163983223936/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321064 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321048.014984:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983223936:0@lo:400:kworker.0 00000100:00100000:3.0:1767321048.019829:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321048.019876:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983224320:0@lo:38:kworker.0 00000100:00100000:3.0:1767321048.019907:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321048.019940:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983224448:0@lo:38:kworker.0 00000100:00100000:3.0:1767321048.019966:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321048.019988:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983224832:0@lo:38:kworker.0 00000100:00100000:3.0:1767321048.020009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321048.020070:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224064 00000100:00100000:3.0:1767321048.020079:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224064 00000100:00100000:3.0:1767321048.020108:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224320 00000100:00100000:3.0:1767321048.020111:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224320 00000100:00100000:3.0:1767321048.020120:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224832 00000100:00100000:3.0:1767321048.020122:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224832 00000100:00100000:3.0:1767321048.020134:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2753 00000100:00100000:3.0:1767321048.020141:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224064:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321048.020160:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321048.020201:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f960700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224064:12345-0@lo:38:kworker.0 Request processed in 59us (380us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321048.020208:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2753 00000100:00100000:3.0:1767321048.020224:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2754 00000100:00100000:3.0:1767321048.020227:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224320:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321048.020235:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321048.020258:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f962680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224320:12345-0@lo:38:kworker.0 Request processed in 29us (351us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321048.020264:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2754 00000100:00100000:3.0:1767321048.020295:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2755 00000100:00100000:3.0:1767321048.020300:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224832:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321048.020309:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321048.020333:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983224832:12345-0@lo:38:kworker.0 Request processed in 32us (323us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321048.020338:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2755 00000100:00100000:3.0:1767321048.020355:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224448 00000100:00100000:3.0:1767321048.020359:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224448 00000100:00100000:3.0:1767321048.020377:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 713 00000100:00100000:3.0:1767321048.020382:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983224448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321048.020391:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321048.020417:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460543ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983224448:12345-0@lo:38:kworker.0 Request processed in 34us (450us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321048.020423:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 713 00000100:00080000:3.0:1767321048.020463:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321048.020468:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321048.020472:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983224064:0@lo:38:kworker.0 00000100:00080000:3.0:1767321048.020480:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321048.020484:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321048.020486:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983224320:0@lo:38:kworker.0 00000100:00080000:3.0:1767321048.020493:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321048.020495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321048.020498:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983224448:0@lo:38:kworker.0 00000100:00080000:3.0:1767321048.020504:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321048.020507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321048.020509:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983224832:0@lo:38:kworker.0 00000100:00100000:1.0:1767321048.023848:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983224192:12345-0@lo:400:kworker.0 Request processed in 18723us (18966us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321048.023863:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2079 00010000:00080000:0.0:1767321048.024003:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346063b2d80 x1853163983224192/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321064 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321048.024018:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983224192:0@lo:400:kworker.0 00010000:00010000:3.0:1767321048.197140:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321048.973048:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321048.973403:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321048.973505:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715968 00000100:00100000:1.0:1767321048.973518:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715968 00000100:00100000:1.0:1767321048.973612:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915715840 00000100:00100000:1.0:1767321048.973615:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915715840 00000100:00100000:1.0:1767321048.973638:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2081 00000100:00100000:1.0:1767321048.973648:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715968:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321048.973731:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915715968:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (694us total) trans 0 rc 0/0 00000100:00080000:0.0:1767321048.973733:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321048.973742:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2081 00000100:00100000:1.0:1767321048.973766:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2756 00000100:00100000:1.0:1767321048.973772:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915715840:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321048.973792:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321048.973848:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915715840:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 75us (445us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321048.973855:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2756 00010000:00010000:3.0:1767321049.220851:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321050.244893:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321051.271248:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321051.588790:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321051.588815:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983224960:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321051.588887:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321051.589014:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983224960 00000100:00100000:3.0:1767321051.589023:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983224960 00000100:00100000:3.0:1767321051.589107:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1388 00000100:00100000:3.0:1767321051.589118:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983224960:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321051.589152:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321051.589202:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983224960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321051.589209:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1388 00000100:00100000:3.0:1767321051.589253:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983224960:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321051.592177:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321051.592294:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321051.594679:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983225088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321051.594764:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321051.598348:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225088 00000100:00100000:2.0:1767321051.598365:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225088 00000100:00100000:2.0:1767321051.598473:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1389 00000100:00100000:2.0:1767321051.598483:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983225088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321051.598517:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767321051.598696:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321051.599794:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e244e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983225088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 1306us (5024us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321051.599805:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1389 00000100:00100000:0.0:1767321051.601253:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346395e8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983225088:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767321052.292580:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321053.126112:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321053.126124:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321053.126194:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321053.126198:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321053.126202:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321053.126210:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6296 00000100:00080000:2.0:1767321053.126216:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321053.126223:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321053.126247:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469df80 x1853163983225344/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321053.128031:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983225216:0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.128102:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321053.128164:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225216 00000100:00100000:3.0:1767321053.128172:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225216 00000100:00100000:3.0:1767321053.128226:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082 00000100:00100000:3.0:1767321053.128236:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983225216:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.128316:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983225216:12345-0@lo:400:kworker.0 Request processed in 81us (212us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321053.128324:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082 00000100:00080000:3.0:1767321053.128351:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767321053.128380:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469d500 x1853163983225216/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321069 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321053.128394:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983225216:0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.128417:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983225472:0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.128441:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321053.128468:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225472 00000100:00100000:3.0:1767321053.128472:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225472 00000100:00100000:3.0:1767321053.128497:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2083 00000100:00100000:3.0:1767321053.128502:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983225472:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.128534:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ea680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983225472:12345-0@lo:400:kworker.0 Request processed in 32us (93us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321053.128540:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2083 00000100:00080000:3.0:1767321053.128553:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767321053.128570:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460469dc00 x1853163983225472/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321069 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321053.128579:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983225472:0@lo:400:kworker.0 00000100:00100000:0.0:1767321053.128982:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983225344:0@lo:38:kworker.0 00000100:00100000:0.0:1767321053.129017:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321053.129046:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225344 00000100:00100000:0.0:1767321053.129052:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225344 00000100:00100000:0.0:1767321053.129073:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2757 00000100:00100000:0.0:1767321053.129078:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983225344:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321053.129097:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321053.129129:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983225344:12345-0@lo:38:kworker.0 Request processed in 50us (112us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321053.129136:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2757 00000100:00080000:0.0:1767321053.129161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321053.129166:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321053.129169:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983225344:0@lo:38:kworker.0 00000100:00080000:2.0:1767321053.131650:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321053.131660:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321053.131664:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321053.131672:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6296 00000100:00080000:2.0:1767321053.131676:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321053.131685:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321053.131723:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469ea00 x1853163983225600/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321053.131750:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321053.131754:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321053.131756:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321053.131760:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6296 00000100:00080000:2.0:1767321053.131762:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321053.131766:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321053.131781:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d180 x1853163983225728/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321053.134072:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983225600:0@lo:38:kworker.0 00000100:00100000:0.0:1767321053.134150:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321053.134214:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983225728:0@lo:38:kworker.0 00000100:00100000:0.0:1767321053.134241:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321053.134313:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225728 00000100:00100000:0.0:1767321053.134320:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225728 00000100:00100000:0.0:1767321053.134359:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 714 00000100:00100000:0.0:1767321053.134366:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983225728:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321053.134398:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321053.134441:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983225728:12345-0@lo:38:kworker.0 Request processed in 74us (198us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321053.134498:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 714 00000100:00080000:0.0:1767321053.134544:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321053.134549:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321053.134552:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983225728:0@lo:38:kworker.0 00000100:00100000:3.0:1767321053.134705:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225600 00000100:00100000:3.0:1767321053.134710:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225600 00000100:00100000:3.0:1767321053.134742:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2758 00000100:00100000:3.0:1767321053.134747:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983225600:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321053.134757:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321053.134791:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983225600:12345-0@lo:38:kworker.0 Request processed in 43us (641us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321053.134797:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2758 00000100:00080000:0.0:1767321053.135249:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321053.135255:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321053.135259:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983225600:0@lo:38:kworker.0 00000100:00100000:1.0:1767321053.137147:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983225856:0@lo:400:kworker.0 00000100:00100000:1.0:1767321053.137220:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321053.137439:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225856 00000100:00100000:0.0:1767321053.137447:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225856 00000100:00100000:0.0:1767321053.137516:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1277 00000100:00100000:0.0:1767321053.137526:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983225856:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321053.137584:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e247800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983225856:12345-0@lo:400:kworker.0 Request processed in 61us (364us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321053.137592:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1277 00000100:00080000:3.0:1767321053.137756:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321053.137825:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983225856:0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.138161:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983225984:0@lo:400:kworker.0 00000100:00100000:3.0:1767321053.138196:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321053.138258:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983225984 00000100:00100000:0.0:1767321053.138263:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983225984 00000100:00100000:0.0:1767321053.138336:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1278 00000100:00100000:0.0:1767321053.138343:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983225984:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321053.138415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321053.139810:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321053.139818:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321053.139822:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321053.139828:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6296 00000100:00080000:2.0:1767321053.139832:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321053.139838:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321053.139873:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460469d500 x1853163983226112/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321053.140614:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983226112:0@lo:38:kworker.0 00000100:00100000:1.0:1767321053.140673:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321053.140781:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226112 00000100:00100000:2.0:1767321053.140790:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226112 00000100:00100000:2.0:1767321053.140835:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2759 00000100:00100000:2.0:1767321053.140841:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983226112:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321053.140862:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321053.140905:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638596d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983226112:12345-0@lo:38:kworker.0 Request processed in 63us (231us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321053.140911:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2759 00000100:00080000:2.0:1767321053.140960:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321053.140965:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321053.140982:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983226112:0@lo:38:kworker.0 00000100:00100000:0.0:1767321053.141547:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395eb100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983225984:12345-0@lo:400:kworker.0 Request processed in 3202us (3348us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321053.141557:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1278 00000100:00100000:3.0:1767321053.141623:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983225984:0@lo:400:kworker.0 00010000:00010000:2.0:1767321053.316586:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321054.100757:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321054.100855:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321054.101225:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716224 00000100:00100000:0.0:1767321054.101237:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716224 00000100:00100000:0.0:1767321054.101337:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716096 00000100:00100000:0.0:1767321054.101341:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716096 00000100:00100000:0.0:1767321054.101360:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2084 00000100:00100000:0.0:1767321054.101381:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915716224:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321054.101467:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395ead80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915716224:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 96us (720us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321054.101475:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2084 00000100:00100000:0.0:1767321054.101493:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2760 00000100:00100000:0.0:1767321054.101497:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716096:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321054.101518:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321054.101543:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346395e8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716096:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (688us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321054.101550:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2760 00000100:00080000:3.0:1767321054.101810:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321054.340814:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321055.364542:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321056.388572:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321056.708726:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321056.708749:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983226240:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321056.708815:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321056.709050:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226240 00000100:00100000:3.0:1767321056.709058:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226240 00000100:00100000:3.0:1767321056.709127:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1390 00000100:00100000:3.0:1767321056.709147:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983226240:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321056.709181:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321056.709228:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983226240:12345-0@lo:13:osp-pre-0-1.0 Request processed in 82us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321056.709234:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1390 00000100:00080000:0.0:1767321056.709411:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321056.709461:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983226240:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321056.710767:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321056.710784:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983226368:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321056.710841:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321056.710935:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226368 00000100:00100000:1.0:1767321056.710942:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226368 00000100:00100000:1.0:1767321056.710990:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1391 00000100:00100000:1.0:1767321056.710997:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983226368:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321056.711018:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:1.0:1767321056.711021:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321056.711063:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983226368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (221us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321056.711071:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1391 00000100:00100000:2.0:1767321056.711099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983226368:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321057.416076:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321058.244706:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321058.244717:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767321058.244807:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983226496:0@lo:400:kworker.0 00000100:00100000:2.0:1767321058.244891:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321058.244996:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321058.245000:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321058.245004:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321058.245009:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6301 00000100:00080000:1.0:1767321058.245706:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321058.245716:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321058.245744:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934638597800 x1853163983226624/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321058.245796:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321058.245800:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321058.245802:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321058.245806:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6301 00000100:00080000:1.0:1767321058.245809:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321058.245812:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321058.245820:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321058.245830:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983226752:0@lo:400:kworker.0 00000100:00100000:1.0:1767321058.245882:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321058.245925:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983226624:0@lo:38:kworker.0 00000100:00100000:0.0:1767321058.246041:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346063b0a80 x1853163983226880/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321058.246063:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321058.246066:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321058.246068:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321058.246072:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6301 00000100:00080000:0.0:1767321058.246075:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321058.246079:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321058.246092:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911500 x1853163983227008/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321058.246116:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321058.246124:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983227136:0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.246156:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321058.246206:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227136 00000100:00100000:0.0:1767321058.246212:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227136 00000100:00100000:0.0:1767321058.246268:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1279 00000100:00100000:0.0:1767321058.246308:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983227136:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.246371:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983227136:12345-0@lo:400:kworker.0 Request processed in 66us (214us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321058.246379:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1279 00000100:00100000:0.0:1767321058.246468:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983227136:0@lo:400:kworker.0 00000100:00080000:0.0:1767321058.246503:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321058.246507:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321058.246508:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321058.246512:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6301 00000100:00080000:0.0:1767321058.246515:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321058.246518:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321058.246527:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983227264:0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.246552:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321058.246575:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227264 00000100:00100000:0.0:1767321058.246579:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227264 00000100:00100000:0.0:1767321058.246603:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1280 00000100:00100000:0.0:1767321058.246610:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983227264:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.246644:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983227264:12345-0@lo:400:kworker.0 Request processed in 35us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321058.246651:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1280 00000100:00100000:0.0:1767321058.246676:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983227264:0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.246699:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911f80 x1853163983227392/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321058.247903:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321058.247912:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321058.257034:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321058.257130:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226496 00000100:00100000:3.0:1767321058.257140:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226496 00000100:00100000:3.0:1767321058.258234:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226752 00000100:00100000:3.0:1767321058.258243:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226752 00000100:00100000:3.0:1767321058.258289:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085 00000100:00100000:3.0:1767321058.258299:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983226496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321058.258384:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983226496:12345-0@lo:400:kworker.0 Request processed in 86us (13491us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321058.258391:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085 00000100:00100000:3.0:1767321058.258409:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2086 00000100:00100000:3.0:1767321058.258415:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983226752:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321058.258564:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983226752:12345-0@lo:400:kworker.0 Request processed in 148us (12681us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321058.258571:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2086 00010000:00080000:3.0:1767321058.258640:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934638594e00 x1853163983226496/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321074 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321058.258654:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638594e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983226496:0@lo:400:kworker.0 00000100:00080000:1.0:1767321058.261828:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321058.261901:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93462dee4700 x1853163983226752/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321074 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321058.261918:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462dee4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983226752:0@lo:400:kworker.0 00000100:00100000:0.0:1767321058.266969:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346063b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983226880:0@lo:38:kworker.0 00000100:00100000:0.0:1767321058.267042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321058.272998:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983227008:0@lo:38:kworker.0 00000100:00100000:0.0:1767321058.273072:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321058.273104:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227008 00000100:00100000:0.0:1767321058.273112:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227008 00000100:00100000:0.0:1767321058.273155:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 715 00000100:00100000:0.0:1767321058.273164:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983227008:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321058.273187:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321058.273373:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226624 00000100:00100000:2.0:1767321058.273380:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226624 00000100:00100000:2.0:1767321058.273405:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983226880 00000100:00100000:2.0:1767321058.273409:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983226880 00000100:00100000:2.0:1767321058.273430:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2761 00000100:00100000:2.0:1767321058.273435:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983226624:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321058.273445:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321058.273491:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631909880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983226624:12345-0@lo:38:kworker.0 Request processed in 54us (16473us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321058.273498:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2761 00000100:00100000:2.0:1767321058.273514:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2762 00000100:00100000:2.0:1767321058.273518:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983226880:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321058.273526:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321058.273549:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983226880:12345-0@lo:38:kworker.0 Request processed in 30us (6507us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321058.273555:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2762 00000100:00100000:0.0:1767321058.274811:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934631908e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983227008:12345-0@lo:38:kworker.0 Request processed in 1643us (1735us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321058.274821:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 715 00000100:00080000:0.0:1767321058.274895:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321058.274901:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321058.274905:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983227008:0@lo:38:kworker.0 00000100:00100000:0.0:1767321058.274914:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983227392:0@lo:38:kworker.0 00000100:00100000:0.0:1767321058.274949:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321058.274975:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321058.274979:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321058.274983:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934638597800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983226624:0@lo:38:kworker.0 00000100:00080000:0.0:1767321058.275023:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321058.275028:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321058.275031:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346063b0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983226880:0@lo:38:kworker.0 00000100:00100000:2.0:1767321058.280663:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227392 00000100:00100000:2.0:1767321058.280679:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227392 00000100:00100000:2.0:1767321058.280737:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2763 00000100:00100000:2.0:1767321058.280745:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983227392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321058.280769:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321058.280864:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983227392:12345-0@lo:38:kworker.0 Request processed in 118us (5913us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321058.280871:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2763 00000100:00080000:0.0:1767321058.280992:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321058.280999:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321058.281003:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983227392:0@lo:38:kworker.0 00010000:00010000:3.0:1767321058.436546:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321059.212931:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321059.212981:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321059.213150:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716352 00000100:00100000:3.0:1767321059.213161:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716352 00000100:00100000:3.0:1767321059.213202:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716480 00000100:00100000:3.0:1767321059.213205:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716480 00000100:00100000:3.0:1767321059.213245:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2087 00000100:00100000:3.0:1767321059.213254:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915716480:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321059.213304:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2764 00000100:00100000:2.0:1767321059.213311:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716352:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321059.213333:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321059.213459:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346063b0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915716480:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 205us (476us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321059.213467:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2087 00000100:00100000:2.0:1767321059.213520:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463190a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716352:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 206us (599us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321059.213527:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2764 00000100:00080000:1.0:1767321059.213649:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321059.460471:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321060.488381:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321061.508533:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321061.828664:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321061.829063:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983227648:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321061.829139:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321061.829305:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227648 00000100:00100000:3.0:1767321061.829315:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227648 00000100:00100000:3.0:1767321061.829387:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1392 00000100:00100000:3.0:1767321061.829397:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983227648:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321061.829429:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321061.829480:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983227648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 84us (341us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321061.829488:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1392 00000100:00100000:3.0:1767321061.829536:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983227648:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321061.829655:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321061.830091:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983227520:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321061.830130:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321061.830254:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227520 00000100:00100000:2.0:1767321061.830261:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227520 00000100:00100000:2.0:1767321061.830389:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1393 00000100:00100000:2.0:1767321061.830397:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983227520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321061.830418:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321061.830465:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983227520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (334us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321061.830472:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1393 00000100:00100000:1.0:1767321061.830547:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983227520:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321061.830627:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321062.533420:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321063.367593:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321063.367605:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767321063.367727:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983227776:0@lo:400:kworker.0 00000100:00100000:2.0:1767321063.367826:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321063.367956:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227776 00000100:00100000:1.0:1767321063.367965:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227776 00000100:00100000:1.0:1767321063.368033:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088 00000100:00100000:1.0:1767321063.368044:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983227776:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321063.368410:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321063.370136:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321063.370146:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321063.370150:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321063.370156:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6306 00000100:00080000:0.0:1767321063.370161:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321063.370168:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321063.370223:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d880 x1853163983227904/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321063.370268:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321063.370288:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321063.370290:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321063.370294:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6306 00000100:00080000:0.0:1767321063.370296:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321063.370300:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321063.370312:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396c380 x1853163983228160/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321063.370321:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321063.370324:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321063.370326:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321063.370329:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6306 00000100:00080000:0.0:1767321063.370331:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321063.370334:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321063.370345:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163983228288/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321063.370462:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983228544:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.370522:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321063.370595:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983228416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.370619:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321063.370640:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321063.370645:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983228032:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.370670:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321063.370694:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228032 00000100:00100000:3.0:1767321063.370700:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228032 00000100:00100000:3.0:1767321063.370758:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2089 00000100:00100000:3.0:1767321063.370768:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983228032:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.370815:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983228032:12345-0@lo:400:kworker.0 Request processed in 48us (143us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321063.370822:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2089 00000100:00080000:3.0:1767321063.370842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767321063.370866:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463396dc00 x1853163983228032/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321079 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321063.370878:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983228032:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.370933:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228544 00000100:00100000:3.0:1767321063.370937:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228544 00000100:00100000:3.0:1767321063.370967:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228416 00000100:00100000:3.0:1767321063.370970:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228416 00000100:00100000:3.0:1767321063.370990:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1281 00000100:00100000:3.0:1767321063.370997:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983228544:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.371032:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983228544:12345-0@lo:400:kworker.0 Request processed in 35us (509us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321063.371038:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1281 00000100:00100000:3.0:1767321063.371050:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1282 00000100:00100000:3.0:1767321063.371056:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983228416:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.371082:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983228416:12345-0@lo:400:kworker.0 Request processed in 27us (462us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321063.371088:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1282 00000100:00100000:3.0:1767321063.371107:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983227904:0@lo:38:kworker.0 00000100:00100000:3.0:1767321063.371130:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321063.371152:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983228160:0@lo:38:kworker.0 00000100:00100000:3.0:1767321063.371169:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321063.371184:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983228288:0@lo:38:kworker.0 00000100:00100000:3.0:1767321063.371204:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321063.371232:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321063.371235:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321063.371241:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983227904 00000100:00100000:3.0:1767321063.371244:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983227904 00000100:00100000:3.0:1767321063.371268:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228160 00000100:00100000:3.0:1767321063.371293:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228160 00000100:00100000:3.0:1767321063.371304:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2765 00000100:00100000:3.0:1767321063.371308:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983227904:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321063.371326:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321063.371359:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983227904:12345-0@lo:38:kworker.0 Request processed in 49us (228us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321063.371365:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2765 00000100:00100000:3.0:1767321063.371376:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2766 00000100:00100000:3.0:1767321063.371379:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983228160:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321063.371388:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321063.371410:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983228160:12345-0@lo:38:kworker.0 Request processed in 29us (240us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321063.371416:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2766 00000100:00080000:3.0:1767321063.371438:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321063.371443:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321063.371447:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983227904:0@lo:38:kworker.0 00000100:00080000:3.0:1767321063.371455:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321063.371458:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321063.371461:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983228160:0@lo:38:kworker.0 00000100:00100000:3.0:1767321063.371489:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983228544:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.371513:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983228416:0@lo:400:kworker.0 00000100:00100000:3.0:1767321063.371565:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228288 00000100:00100000:3.0:1767321063.371568:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228288 00000100:00100000:3.0:1767321063.371587:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 716 00000100:00100000:3.0:1767321063.371591:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983228288:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321063.371601:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321063.371630:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983228288:12345-0@lo:38:kworker.0 Request processed in 37us (425us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321063.371636:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 716 00000100:00080000:3.0:1767321063.371661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321063.371665:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321063.371668:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983228288:0@lo:38:kworker.0 00000100:00100000:1.0:1767321063.375347:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983227776:12345-0@lo:400:kworker.0 Request processed in 7293us (7509us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321063.375363:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088 00010000:00080000:2.0:1767321063.376761:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911500 x1853163983227776/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321079 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321063.376780:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983227776:0@lo:400:kworker.0 00000100:00080000:0.0:1767321063.378227:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321063.378236:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321063.378240:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321063.378246:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6306 00000100:00080000:0.0:1767321063.378251:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321063.378257:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321063.378322:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ded80 x1853163983228672/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321063.379080:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983228672:0@lo:38:kworker.0 00000100:00100000:3.0:1767321063.379161:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321063.383193:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228672 00000100:00100000:1.0:1767321063.383210:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228672 00000100:00100000:1.0:1767321063.383342:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2767 00000100:00100000:1.0:1767321063.383351:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983228672:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321063.383375:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321063.383472:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983228672:12345-0@lo:38:kworker.0 Request processed in 120us (4311us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321063.383479:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2767 00000100:00080000:2.0:1767321063.383573:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321063.383579:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321063.383583:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983228672:0@lo:38:kworker.0 00010000:00010000:3.0:1767321063.557188:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321064.331420:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321064.331593:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716736 00000100:00100000:2.0:1767321064.331613:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716736 00000100:00100000:2.0:1767321064.331759:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2090 00000100:00100000:2.0:1767321064.331771:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915716736:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321064.331865:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915716736:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 97us (468us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321064.331873:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2090 00000100:00080000:3.0:1767321064.333629:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321064.336378:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767321064.336470:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915716864 offset 0 length 520: 4 00000100:00100000:3.0:1767321064.339083:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716608 00000100:00100000:3.0:1767321064.339098:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716608 00000100:00100000:3.0:1767321064.339161:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2768 00000100:00100000:3.0:1767321064.339170:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716608:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321064.339199:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321064.339319:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915716608:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 148us (2951us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321064.339326:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2768 00010000:00010000:3.0:1767321064.580541:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321065.605726:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321066.628454:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321066.951300:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321066.951329:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983228800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321066.951428:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321066.951697:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228800 00000100:00100000:3.0:1767321066.951706:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228800 00000100:00100000:3.0:1767321066.951774:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1394 00000100:00100000:3.0:1767321066.951782:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983228800:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321066.951820:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321066.951867:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983228800:12345-0@lo:13:osp-pre-0-1.0 Request processed in 86us (440us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321066.951874:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1394 00000100:00080000:0.0:1767321066.952098:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321066.952158:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983228800:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321066.955206:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321066.955230:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983228928:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321066.955337:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321066.955465:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983228928 00000100:00100000:2.0:1767321066.955474:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983228928 00000100:00100000:2.0:1767321066.955548:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1395 00000100:00100000:2.0:1767321066.955556:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983228928:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321066.955588:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321066.955641:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983228928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (304us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321066.955650:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1395 00000100:00100000:2.0:1767321066.955718:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460469ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983228928:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321066.956095:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321067.460413:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321042/real 1767321042] req@ffff93462e246300 x1853163983222528/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321067 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321067.460455:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93462e246300 x1853163983222528/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321067 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321067.460476:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321067.460505:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321067.460509:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93462e246300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983222528:0@lo:250:kworker.0 00000100:00080000:0.0:1767321067.460809:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321067.460815:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321067.460818:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321067.460822:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6291 00000100:00080000:0.0:1767321067.460827:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321067.460833:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321067.460870:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dc380 x1853163983229056/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321067.460938:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321067.460942:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321067.460944:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321067.460948:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6311 00000100:00080000:0.0:1767321067.460950:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321067.460954:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321067.460969:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163983229312/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321067.460997:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321067.461002:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321067.461004:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321067.461007:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6311 00000100:00080000:0.0:1767321067.461010:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321067.461014:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321067.461024:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a0000 x1853163983229568/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321067.461032:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321067.461035:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321067.461036:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321067.461040:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6311 00000100:00080000:0.0:1767321067.461041:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321067.461045:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321067.461055:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a1c00 x1853163983229696/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321067.461064:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321067.461067:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321067.461069:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321067.461072:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6311 00000100:00080000:0.0:1767321067.461074:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321067.461077:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321067.461087:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346333a3100 x1853163983229824/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321067.461113:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983229440:0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.461174:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321067.461349:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321067.461361:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983229184:0@lo:400:kworker.0 00000100:00100000:2.0:1767321067.461418:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983229056:0@lo:250:kworker.0 00000100:00100000:0.0:1767321067.461535:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229440 00000100:00100000:0.0:1767321067.461543:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229440 00000100:00100000:0.0:1767321067.461609:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2091 00000100:00100000:0.0:1767321067.461619:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983229440:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321067.461739:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321067.462957:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000400:00000100:2.0:1767321067.463360:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983229056 offset 0 length 520: 4 00000100:00100000:2.0:1767321067.463385:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983229312:0@lo:38:kworker.0 00000100:00100000:2.0:1767321067.463421:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.463532:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983229440:12345-0@lo:400:kworker.0 Request processed in 1904us (2347us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321067.463542:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2091 00010000:00080000:3.0:1767321067.463681:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1880 x1853163983229440/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321083 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321067.463694:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983229440:0@lo:400:kworker.0 00000100:00100000:2.0:1767321067.463909:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983229568:0@lo:38:kworker.0 00000100:00100000:2.0:1767321067.463936:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.464074:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229184 00000100:00100000:0.0:1767321067.464081:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229184 00000100:00100000:0.0:1767321067.464122:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229312 00000100:00100000:0.0:1767321067.464126:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229312 00000100:00100000:0.0:1767321067.464144:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2092 00000100:00100000:0.0:1767321067.464152:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983229184:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321067.464204:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983229184:12345-0@lo:400:kworker.0 Request processed in 52us (1251us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321067.464210:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2092 00000100:00100000:0.0:1767321067.464237:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2769 00000100:00100000:0.0:1767321067.464242:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983229312:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321067.464264:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321067.464315:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983229312:12345-0@lo:38:kworker.0 Request processed in 71us (892us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321067.464321:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2769 00000100:00080000:0.0:1767321067.464369:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767321067.464410:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4e00 x1853163983229184/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321083 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321067.464421:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983229184:0@lo:400:kworker.0 00000100:00100000:2.0:1767321067.464805:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983229696:0@lo:38:kworker.0 00000100:00100000:2.0:1767321067.464839:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321067.464869:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983229824:0@lo:38:kworker.0 00000100:00100000:2.0:1767321067.464966:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321067.465006:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229696 00000100:00100000:0.0:1767321067.465012:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229696 00000100:00100000:0.0:1767321067.465036:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 717 00000100:00100000:0.0:1767321067.465041:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983229696:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321067.465053:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321067.465082:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469dc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983229696:12345-0@lo:38:kworker.0 Request processed in 40us (242us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321067.465089:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 717 00000100:00080000:2.0:1767321067.465206:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321067.465319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321067.465323:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983229312:0@lo:38:kworker.0 00000100:00080000:2.0:1767321067.465337:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321067.465341:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321067.465344:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983229696:0@lo:38:kworker.0 00000100:00100000:3.0:1767321067.465404:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229568 00000100:00100000:3.0:1767321067.465409:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229568 00000100:00100000:3.0:1767321067.465439:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229824 00000100:00100000:3.0:1767321067.465442:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229824 00000100:00100000:3.0:1767321067.465459:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2770 00000100:00100000:3.0:1767321067.465464:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983229568:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321067.465474:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321067.465505:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460469d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983229568:12345-0@lo:38:kworker.0 Request processed in 40us (1568us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321067.465511:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2770 00000100:00100000:1.0:1767321067.465736:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2771 00000100:00100000:1.0:1767321067.465743:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983229824:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321067.465753:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321067.465785:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462e246300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983229824:12345-0@lo:38:kworker.0 Request processed in 42us (818us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321067.465791:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2771 00000100:00080000:2.0:1767321067.466000:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321067.466005:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321067.466009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983229568:0@lo:38:kworker.0 00000100:00080000:2.0:1767321067.466017:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321067.466020:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321067.466023:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983229824:0@lo:38:kworker.0 00010000:00010000:3.0:1767321067.652480:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321068.676629:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321069.453030:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321069.453114:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321069.453207:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717120 00000100:00100000:0.0:1767321069.453218:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717120 00000100:00100000:0.0:1767321069.453320:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915716992 00000100:00100000:0.0:1767321069.453323:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915716992 00000100:00100000:0.0:1767321069.453344:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2093 00000100:00100000:0.0:1767321069.453354:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915717120:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321069.453437:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915717120:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (418us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321069.453445:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2093 00000100:00100000:0.0:1767321069.453474:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2772 00000100:00100000:0.0:1767321069.453480:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915716992:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321069.453501:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321069.453526:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915716992:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 45us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321069.453539:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2772 00000100:00080000:3.0:1767321069.453665:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321069.700870:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321070.724518:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321071.748485:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321072.070530:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321072.070560:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983229952:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321072.070586:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983230080:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321072.071019:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.071404:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230080 00000100:00100000:1.0:1767321072.071413:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230080 00000100:00100000:1.0:1767321072.071463:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1396 00000100:00100000:1.0:1767321072.071470:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983230080:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321072.071496:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321072.071537:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983230080:12345-0@lo:13:osp-pre-0-1.0 Request processed in 69us (520us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321072.071542:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1396 00000100:00100000:1.0:1767321072.071597:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983230080:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321072.072556:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321072.073263:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321072.074304:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983229952 00000100:00100000:2.0:1767321072.074327:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983229952 00000100:00100000:2.0:1767321072.074406:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1397 00000100:00100000:2.0:1767321072.074417:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983229952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321072.074451:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767321072.074453:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321072.074522:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983229952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (1261us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321072.074528:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1397 00000100:00100000:0.0:1767321072.074571:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983229952:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321072.580724:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321072.580736:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321072.580800:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321072.580804:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321072.580808:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321072.580817:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6315 00000100:00080000:2.0:1767321072.580822:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321072.580829:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321072.580853:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35880 x1853163983230336/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321072.580890:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1767321072.580900:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321072.580905:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321072.580907:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767321072.580909:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983230208:0@lo:400:kworker.0 00000100:00080000:2.0:1767321072.580913:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6315 00000100:00080000:2.0:1767321072.580918:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321072.580923:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321072.580940:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604581f80 x1853163983230592/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321072.580949:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321072.580952:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321072.580954:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321072.580957:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6315 00000100:00080000:2.0:1767321072.580959:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321072.580962:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321072.580974:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604583480 x1853163983230720/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321072.580985:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321072.581001:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321072.581005:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321072.581019:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321072.581024:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6315 00000100:00080000:2.0:1767321072.581027:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767321072.581028:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983230976:0@lo:400:kworker.0 00000100:00080000:2.0:1767321072.581033:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321072.581048:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604582d80 x1853163983231104/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321072.581060:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321072.581092:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230208 00000100:00100000:3.0:1767321072.581098:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230976 00000100:00100000:2.0:1767321072.581100:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230208 00000100:00100000:3.0:1767321072.581103:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230976 00000100:00100000:3.0:1767321072.581154:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1283 00000100:00100000:2.0:1767321072.581155:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094 00000100:00100000:3.0:1767321072.581168:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983230976:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321072.581169:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983230208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.581255:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983230848:0@lo:400:kworker.0 00000100:00100000:0.0:1767321072.581338:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983230336:0@lo:38:kworker.0 00000100:00100000:2.0:1767321072.584858:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983230208:12345-0@lo:400:kworker.0 Request processed in 3683us (3864us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321072.584874:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094 00000100:00080000:2.0:1767321072.584918:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321072.584947:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983230976:12345-0@lo:400:kworker.0 Request processed in 3780us (3884us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321072.584957:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1283 00000100:00100000:2.0:1767321072.585187:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983230976:0@lo:400:kworker.0 00010000:00080000:2.0:1767321072.585204:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911500 x1853163983230208/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321088 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321072.585215:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983230208:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.585394:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.585428:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321072.585438:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983230464:0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.585473:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321072.585512:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230464 00000100:00100000:1.0:1767321072.585702:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230464 00000100:00100000:1.0:1767321072.585770:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2095 00000100:00100000:1.0:1767321072.585779:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983230464:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321072.585833:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983230464:12345-0@lo:400:kworker.0 Request processed in 55us (357us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321072.585839:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2095 00000100:00080000:1.0:1767321072.585857:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321072.585875:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320e00 x1853163983230464/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321088 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321072.585885:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983230464:0@lo:400:kworker.0 00000100:00100000:0.0:1767321072.586034:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321072.586063:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983230592:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.586083:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321072.586101:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230336 00000100:00100000:0.0:1767321072.586104:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230336 00000100:00100000:0.0:1767321072.586121:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230592 00000100:00100000:0.0:1767321072.586124:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230592 00000100:00100000:0.0:1767321072.586139:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2773 00000100:00100000:0.0:1767321072.586144:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983230336:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321072.586163:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321072.586189:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983230336:12345-0@lo:38:kworker.0 Request processed in 44us (155us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321072.586195:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2773 00000100:00100000:0.0:1767321072.586206:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2774 00000100:00100000:0.0:1767321072.586210:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983230592:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321072.586217:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321072.586237:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983230592:12345-0@lo:38:kworker.0 Request processed in 26us (153us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321072.586242:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2774 00000100:00080000:0.0:1767321072.586267:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.586287:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.586291:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983230592:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.586311:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983230720:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.586339:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321072.586363:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230720 00000100:00100000:0.0:1767321072.586367:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230720 00000100:00100000:0.0:1767321072.586389:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 718 00000100:00100000:0.0:1767321072.586395:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983230720:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321072.586405:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983230848 00010000:00020000:0.0:1767321072.586406:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321072.586414:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983230848 00000100:00100000:0.0:1767321072.586437:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983230720:12345-0@lo:38:kworker.0 Request processed in 40us (96us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321072.586443:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 718 00000100:00100000:3.0:1767321072.586452:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1284 00000100:00100000:3.0:1767321072.586462:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983230848:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321072.586476:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.586481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.586485:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983230720:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.586491:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983231104:0@lo:38:kworker.0 00000100:00100000:0.0:1767321072.586512:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321072.586543:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231104 00000100:00100000:0.0:1767321072.586547:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231104 00000100:00100000:0.0:1767321072.586565:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2775 00000100:00100000:0.0:1767321072.586570:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983231104:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321072.586593:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321072.586617:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983231104:12345-0@lo:38:kworker.0 Request processed in 46us (104us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321072.586623:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2775 00000100:00080000:0.0:1767321072.586643:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.586646:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.586650:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604582d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983231104:0@lo:38:kworker.0 00000100:00080000:0.0:1767321072.586672:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321072.586677:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321072.586680:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983230336:0@lo:38:kworker.0 00000100:00100000:3.0:1767321072.587419:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93462dee7b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983230848:12345-0@lo:400:kworker.0 Request processed in 957us (2022us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321072.587429:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1284 00000100:00100000:1.0:1767321072.587491:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604583100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983230848:0@lo:400:kworker.0 00010000:00010000:3.0:1767321072.772871:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321073.797533:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321074.572427:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321074.572474:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321074.572627:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717376 00000100:00100000:0.0:1767321074.572638:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717376 00000100:00100000:3.0:1767321074.572640:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717248 00000100:00100000:3.0:1767321074.572643:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717248 00000100:00100000:3.0:1767321074.572686:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2096 00000100:00100000:0.0:1767321074.572692:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2776 00000100:00100000:3.0:1767321074.572696:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717376:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321074.572697:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915717248:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321074.572717:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321074.572939:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915717248:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 238us (462us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321074.572947:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2776 00000100:00080000:1.0:1767321074.573053:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321074.573194:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717376:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 499us (778us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321074.573201:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2096 00010000:00010000:2.0:1767321074.820589:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321075.844510:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321076.868548:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321077.188490:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983231232:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321077.188571:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321077.188679:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983231360:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321077.188708:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321077.188776:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231232 00000100:00100000:2.0:1767321077.188785:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231232 00000100:00100000:2.0:1767321077.188846:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231360 00000100:00100000:2.0:1767321077.188849:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231360 00000100:00100000:2.0:1767321077.188866:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1398 00000100:00100000:2.0:1767321077.188875:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983231232:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321077.188904:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321077.188953:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983231232:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (382us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321077.188961:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1398 00000100:00100000:2.0:1767321077.188977:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1399 00000100:00100000:2.0:1767321077.188982:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983231360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321077.189000:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767321077.189080:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321077.189085:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321077.189169:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983231232:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321077.189383:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983231360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 400us (673us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321077.189390:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1399 00000100:00100000:1.0:1767321077.189444:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604581c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983231360:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321077.700409:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321077.700430:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321077.700497:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321077.700501:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321077.700505:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321077.700514:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6321 00000100:00080000:1.0:1767321077.700526:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321077.700534:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321077.700557:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460435ad80 x1853163983231616/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321077.700603:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321077.700607:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321077.700609:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321077.700613:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6321 00000100:00080000:1.0:1767321077.700616:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321077.700620:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321077.700654:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fea00 x1853163983231872/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321077.700665:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321077.700669:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321077.700671:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767321077.700675:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983231616:0@lo:38:kworker.0 00000100:00080000:1.0:1767321077.700676:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6321 00000100:00080000:1.0:1767321077.700694:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321077.700699:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321077.700713:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff100 x1853163983232000/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321077.700738:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321077.700742:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321077.700744:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321077.700748:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6321 00000100:00100000:3.0:1767321077.700754:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321077.700755:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321077.700762:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321077.700775:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc000 x1853163983232384/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321077.700798:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231616 00000100:00100000:3.0:1767321077.700805:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231616 00000100:00100000:1.0:1767321077.700816:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983231744:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.700852:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2777 00000100:00100000:3.0:1767321077.700860:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983231616:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321077.700867:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00020000:3.0:1767321077.700886:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321077.700894:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321077.700901:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983232128:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.700932:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983231616:12345-0@lo:38:kworker.0 Request processed in 71us (180us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321077.700939:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.700947:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2777 00000100:00100000:3.0:1767321077.700966:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231744 00000100:00100000:3.0:1767321077.700970:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231744 00000100:00100000:1.0:1767321077.700990:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232128 00000100:00100000:1.0:1767321077.700996:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232128 00000100:00100000:3.0:1767321077.701008:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097 00000100:00100000:3.0:1767321077.701017:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983231744:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321077.701032:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1285 00000100:00100000:1.0:1767321077.701042:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983232128:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.701063:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983231744:12345-0@lo:400:kworker.0 Request processed in 47us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321077.701071:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097 00000100:00100000:1.0:1767321077.701084:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdc00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983232128:12345-0@lo:400:kworker.0 Request processed in 43us (146us total) trans 0 rc 0/0 00000100:00080000:3.0:1767321077.701090:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321077.701094:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1285 00000100:00080000:3.0:1767321077.701119:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321077.701124:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321077.701129:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983231616:0@lo:38:kworker.0 00000100:00100000:1.0:1767321077.701136:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983232128:0@lo:400:kworker.0 00010000:00080000:1.0:1767321077.701151:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604320e00 x1853163983231744/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321093 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321077.701154:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983231872:0@lo:38:kworker.0 00000100:00100000:1.0:1767321077.701169:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983231744:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.701185:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.701207:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231872 00000100:00100000:3.0:1767321077.701210:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231872 00000100:00100000:3.0:1767321077.701236:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2778 00000100:00100000:3.0:1767321077.701241:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983231872:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321077.701249:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983231488:0@lo:400:kworker.0 00010000:00020000:3.0:1767321077.701252:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321077.701318:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983231872:12345-0@lo:38:kworker.0 Request processed in 72us (131us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321077.701319:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.701328:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2778 00000100:00100000:3.0:1767321077.701339:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983231488 00000100:00100000:2.0:1767321077.701340:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321077.701342:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983231488 00000100:00100000:2.0:1767321077.701348:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983232256:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.701371:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2098 00000100:00100000:2.0:1767321077.701376:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.701382:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983231488:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321077.701410:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232256 00000100:00100000:2.0:1767321077.701415:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232256 00000100:00100000:3.0:1767321077.701421:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983231488:12345-0@lo:400:kworker.0 Request processed in 40us (105us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321077.701428:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2098 00000100:00080000:3.0:1767321077.701444:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321077.701445:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1286 00000100:00100000:2.0:1767321077.701456:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983232256:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321077.701462:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321077.701466:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321077.701470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983231872:0@lo:38:kworker.0 00000100:00100000:3.0:1767321077.701477:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983232000:0@lo:38:kworker.0 00000100:00100000:2.0:1767321077.701504:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983232256:12345-0@lo:400:kworker.0 Request processed in 50us (129us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321077.701511:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1286 00000100:00100000:2.0:1767321077.701536:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983232256:0@lo:400:kworker.0 00010000:00080000:2.0:1767321077.701545:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934604359180 x1853163983231488/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321093 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321077.701553:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983231488:0@lo:400:kworker.0 00000100:00100000:3.0:1767321077.701619:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321077.702098:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983232384:0@lo:38:kworker.0 00000100:00100000:3.0:1767321077.702126:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321077.702678:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232384 00000100:00100000:1.0:1767321077.702698:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232384 00000100:00100000:1.0:1767321077.702793:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2779 00000100:00100000:1.0:1767321077.702810:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983232384:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321077.702826:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321077.702888:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983232384:12345-0@lo:38:kworker.0 Request processed in 88us (761us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321077.702896:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2779 00000100:00080000:1.0:1767321077.704137:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321077.704146:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321077.704152:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983232384:0@lo:38:kworker.0 00000100:00100000:2.0:1767321077.710245:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232000 00000100:00100000:2.0:1767321077.710263:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232000 00000100:00100000:2.0:1767321077.710348:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 719 00000100:00100000:2.0:1767321077.710359:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983232000:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321077.710384:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321077.710476:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983232000:12345-0@lo:38:kworker.0 Request processed in 118us (8856us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321077.710493:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 719 00000100:00080000:1.0:1767321077.710689:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321077.710696:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321077.710701:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983232000:0@lo:38:kworker.0 00010000:00010000:2.0:1767321077.892490:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321078.916670:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321079.692056:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321079.692173:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321079.692839:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717504 00000100:00100000:0.0:1767321079.692849:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717504 00000100:00100000:2.0:1767321079.692924:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717632 00000100:00100000:2.0:1767321079.692929:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717632 00000100:00100000:0.0:1767321079.692937:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2780 00000100:00100000:0.0:1767321079.692946:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915717504:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321079.692969:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321079.693320:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915717504:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 372us (1273us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321079.693328:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2780 00000100:00100000:0.0:1767321079.693345:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2099 00000100:00100000:0.0:1767321079.693353:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717632:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:1.0:1767321079.693617:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321079.693666:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717632:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 312us (1492us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321079.693673:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2099 00010000:00010000:3.0:1767321079.940889:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321080.965008:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321081.989502:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321082.308793:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321082.308818:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983232512:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321082.308887:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321082.308930:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321082.308938:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983232640:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321082.310432:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321082.321390:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232512 00000100:00100000:3.0:1767321082.321416:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232512 00000100:00100000:2.0:1767321082.321564:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232640 00000100:00100000:2.0:1767321082.321571:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232640 00000100:00100000:3.0:1767321082.321645:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1400 00000100:00100000:3.0:1767321082.321657:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983232512:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321082.321691:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321082.321801:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983232512:12345-0@lo:13:osp-pre-0-1.0 Request processed in 146us (12912us total) trans 0 rc 0/0 00000100:00080000:1.0:1767321082.324906:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321082.324916:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321082.325019:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983232512:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321082.325759:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1400 00000100:00100000:2.0:1767321082.325884:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1401 00000100:00100000:2.0:1767321082.325899:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983232640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321082.325931:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321082.325997:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983232640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 105us (15570us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321082.326002:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1401 00000100:00100000:1.0:1767321082.327407:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983232640:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321082.823841:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321082.823852:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767321082.823971:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983232768:0@lo:400:kworker.0 00000100:00100000:1.0:1767321082.824045:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321082.824930:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321082.824938:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321082.824942:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321082.824949:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6326 00000100:00080000:0.0:1767321082.824954:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321082.824962:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321082.824996:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609a35880 x1853163983232896/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321082.825332:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983232896:0@lo:38:kworker.0 00000100:00080000:0.0:1767321082.826106:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321082.826112:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321082.826115:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321082.826120:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6326 00000100:00080000:0.0:1767321082.826123:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321082.826137:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321082.826154:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4e00 x1853163983233152/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321082.826170:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321082.826174:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321082.826176:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321082.826180:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6326 00000100:00080000:0.0:1767321082.826183:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321082.826186:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321082.826198:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604375180 x1853163983233280/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321082.826223:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321082.826226:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321082.826228:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321082.826231:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6326 00000100:00080000:0.0:1767321082.826233:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321082.826236:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321082.826246:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983233664/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321082.826290:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983233408:0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.826369:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321082.826462:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321082.826469:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983233024:0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.826511:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233408 00000100:00100000:0.0:1767321082.826517:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233408 00000100:00100000:0.0:1767321082.826563:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1287 00000100:00100000:0.0:1767321082.826582:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983233408:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.826644:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983233408:12345-0@lo:400:kworker.0 Request processed in 64us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321082.826655:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1287 00000100:00100000:0.0:1767321082.826684:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983233536:0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.826721:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321082.826747:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233536 00000100:00100000:0.0:1767321082.826751:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233536 00000100:00100000:0.0:1767321082.826770:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1288 00000100:00100000:0.0:1767321082.826776:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983233536:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.826808:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983233536:12345-0@lo:400:kworker.0 Request processed in 32us (86us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321082.826814:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1288 00000100:00100000:0.0:1767321082.826849:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604376680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983233536:0@lo:400:kworker.0 00000100:00100000:1.0:1767321082.827147:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321082.829219:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983233408:0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.830031:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321082.831026:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232768 00000100:00100000:2.0:1767321082.831038:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232768 00000100:00080000:1.0:1767321082.831933:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321082.832053:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233024 00000100:00100000:2.0:1767321082.832068:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233024 00000100:00100000:2.0:1767321082.832117:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1767321082.832134:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983232768:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321082.832230:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fdf80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983232768:12345-0@lo:400:kworker.0 Request processed in 104us (8184us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321082.832237:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1767321082.832253:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2101 00000100:00100000:2.0:1767321082.832259:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983233024:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321082.833140:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321082.833193:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1880 x1853163983232768/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321098 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321082.833209:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983232768:0@lo:400:kworker.0 00000100:00100000:2.0:1767321082.833894:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983233024:12345-0@lo:400:kworker.0 Request processed in 1631us (6743us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321082.833905:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2101 00010000:00080000:1.0:1767321082.834088:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d7100 x1853163983233024/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321098 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321082.834105:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983233024:0@lo:400:kworker.0 00000100:00100000:0.0:1767321082.834995:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983233152:0@lo:38:kworker.0 00000100:00100000:0.0:1767321082.835043:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321082.835311:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983232896 00000100:00100000:1.0:1767321082.835319:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983232896 00000100:00100000:1.0:1767321082.835354:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233152 00000100:00100000:1.0:1767321082.835357:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233152 00000100:00100000:1.0:1767321082.835373:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2781 00000100:00100000:1.0:1767321082.835381:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983232896:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321082.835410:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321082.835468:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983232896:12345-0@lo:38:kworker.0 Request processed in 87us (5440us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321082.835475:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2781 00000100:00100000:1.0:1767321082.835498:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2782 00000100:00100000:1.0:1767321082.835501:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983233152:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321082.835510:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321082.835533:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983233152:12345-0@lo:38:kworker.0 Request processed in 30us (490us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321082.835538:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2782 00000100:00080000:0.0:1767321082.836414:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321082.836880:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321082.836886:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983233152:0@lo:38:kworker.0 00000100:00100000:0.0:1767321082.836897:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983233280:0@lo:38:kworker.0 00000100:00100000:0.0:1767321082.836950:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321082.836981:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983233664:0@lo:38:kworker.0 00000100:00100000:0.0:1767321082.837003:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321082.837077:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321082.837081:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321082.837095:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983232896:0@lo:38:kworker.0 00000100:00100000:1.0:1767321082.837322:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233280 00000100:00100000:1.0:1767321082.837330:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233280 00000100:00100000:1.0:1767321082.837368:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 720 00000100:00100000:1.0:1767321082.837374:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983233280:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321082.837401:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321082.837453:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983233280:12345-0@lo:38:kworker.0 Request processed in 78us (503us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321082.837460:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 720 00000100:00100000:3.0:1767321082.837723:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233664 00000100:00100000:3.0:1767321082.837729:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233664 00000100:00080000:0.0:1767321082.837796:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321082.837801:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321082.837804:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983233280:0@lo:38:kworker.0 00000100:00100000:3.0:1767321082.839101:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2783 00000100:00100000:3.0:1767321082.839113:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983233664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321082.839140:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321082.839220:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983233664:12345-0@lo:38:kworker.0 Request processed in 107us (2215us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321082.839228:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2783 00000100:00080000:1.0:1767321082.839318:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321082.839323:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321082.839327:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983233664:0@lo:38:kworker.0 00010000:00010000:3.0:1767321083.013783:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321084.041345:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321084.813330:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321084.813480:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321084.814610:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717888 00000100:00100000:3.0:1767321084.814628:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717888 00000100:00100000:3.0:1767321084.814714:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915717760 00000100:00100000:3.0:1767321084.814718:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915717760 00000100:00100000:3.0:1767321084.814752:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2102 00000100:00100000:3.0:1767321084.814764:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717888:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321084.814866:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915717888:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 103us (1546us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321084.814873:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2102 00000100:00100000:3.0:1767321084.814894:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2784 00000100:00100000:3.0:1767321084.814900:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915717760:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321084.814929:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321084.814960:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915717760:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (1487us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321084.814967:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2784 00000100:00080000:1.0:1767321084.822384:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321085.061361:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321086.084600:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321087.109665:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321087.428695:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321087.428720:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983233792:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321087.428807:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321087.428912:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233792 00000100:00100000:1.0:1767321087.428920:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233792 00000100:00100000:1.0:1767321087.428985:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1402 00000100:00100000:1.0:1767321087.428995:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983233792:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321087.429032:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321087.429080:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983233792:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321087.429099:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1402 00000100:00080000:0.0:1767321087.429415:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321087.429444:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983233792:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321087.432142:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983233920:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321087.432211:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321087.432324:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983233920 00000100:00100000:2.0:1767321087.432333:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983233920 00000100:00100000:2.0:1767321087.432403:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1403 00000100:00100000:2.0:1767321087.432412:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983233920:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321087.432439:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:0.0:1767321087.432728:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321087.432936:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ffb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983233920:12345-0@lo:13:osp-pre-0-1.0 Request processed in 525us (724us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321087.432944:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1403 00000100:00100000:1.0:1767321087.433142:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983233920:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321087.942025:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321087.942038:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321087.942105:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321087.942114:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321087.942118:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321087.942126:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6331 00000100:00080000:3.0:1767321087.942131:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321087.942137:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321087.942159:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359180 x1853163983234176/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321087.942176:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983234048:0@lo:400:kworker.0 00000100:00080000:3.0:1767321087.942221:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321087.942225:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321087.942227:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321087.942232:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6331 00000100:00080000:3.0:1767321087.942235:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321087.942239:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321087.942251:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321087.942254:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396fb80 x1853163983234432/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321087.942264:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321087.942268:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321087.942312:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321087.942316:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6331 00000100:00080000:3.0:1767321087.942319:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321087.942324:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321087.942325:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983234176:0@lo:38:kworker.0 00000100:00100000:3.0:1767321087.942338:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396d500 x1853163983234560/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321087.942364:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321087.942367:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321087.942369:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321087.942372:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6331 00000100:00080000:3.0:1767321087.942375:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321087.942378:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321087.942389:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463396dc00 x1853163983234944/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321087.942414:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983234688:0@lo:400:kworker.0 00000100:00100000:0.0:1767321087.942501:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321087.942508:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983234816:0@lo:400:kworker.0 00000100:00100000:3.0:1767321087.942570:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321087.942601:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321087.942605:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234048 00000100:00100000:3.0:1767321087.942608:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983234304:0@lo:400:kworker.0 00000100:00100000:1.0:1767321087.942615:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234048 00000100:00100000:3.0:1767321087.942636:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321087.942677:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234304 00000100:00100000:3.0:1767321087.942682:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234304 00000100:00100000:1.0:1767321087.942686:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2103 00000100:00100000:1.0:1767321087.942698:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983234048:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321087.942720:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2104 00000100:00100000:3.0:1767321087.942729:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983234304:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321087.942758:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983234048:12345-0@lo:400:kworker.0 Request processed in 62us (507us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321087.942766:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2103 00000100:00100000:3.0:1767321087.942774:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638594e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983234304:12345-0@lo:400:kworker.0 Request processed in 46us (136us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321087.942781:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2104 00000100:00100000:1.0:1767321087.942818:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234688 00000100:00100000:1.0:1767321087.942823:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234688 00010000:00080000:3.0:1767321087.942830:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435ad80 x1853163983234304/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321103 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321087.942844:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983234304:0@lo:400:kworker.0 00000100:00100000:1.0:1767321087.942852:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1289 00000100:00100000:1.0:1767321087.942860:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983234688:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321087.942888:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321087.942910:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983234688:0@lo:400:kworker.0 00000100:00100000:1.0:1767321087.942919:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934638597800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983234688:12345-0@lo:400:kworker.0 Request processed in 59us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321087.942927:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1289 00000100:00100000:2.0:1767321087.942951:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321087.942980:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983234432:0@lo:38:kworker.0 00000100:00100000:3.0:1767321087.942990:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234176 00000100:00100000:3.0:1767321087.942996:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234176 00000100:00100000:2.0:1767321087.943007:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321087.943154:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767321087.943179:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934609af2300 x1853163983234048/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321103 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321087.943190:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983234048:0@lo:400:kworker.0 00000100:00100000:0.0:1767321087.943216:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234816 00000100:00100000:0.0:1767321087.943220:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234816 00000100:00100000:0.0:1767321087.943244:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1290 00000100:00100000:0.0:1767321087.943249:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983234816:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321087.943314:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983234816:12345-0@lo:400:kworker.0 Request processed in 64us (159us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321087.943320:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1290 00000100:00100000:0.0:1767321087.943358:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983234816:0@lo:400:kworker.0 00000100:00100000:2.0:1767321087.943453:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983234560:0@lo:38:kworker.0 00000100:00100000:3.0:1767321087.943454:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234432 00000100:00100000:3.0:1767321087.943461:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234432 00000100:00100000:3.0:1767321087.943488:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2785 00000100:00100000:2.0:1767321087.943490:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321087.943497:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983234176:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321087.943516:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983234944:0@lo:38:kworker.0 00010000:00020000:3.0:1767321087.943517:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321087.943557:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321087.943562:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604580380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983234176:12345-0@lo:38:kworker.0 Request processed in 64us (612us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321087.943569:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2785 00000100:00100000:3.0:1767321087.943582:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234944 00000100:00080000:2.0:1767321087.943585:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321087.943586:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234944 00000100:00080000:2.0:1767321087.943593:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321087.943598:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983234176:0@lo:38:kworker.0 00000100:00100000:3.0:1767321087.943604:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2786 00000100:00100000:3.0:1767321087.943610:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983234432:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321087.943620:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321087.943639:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2787 00000100:00100000:2.0:1767321087.943645:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983234944:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321087.943655:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983234432:12345-0@lo:38:kworker.0 Request processed in 42us (647us total) trans 0 rc -19/-19 00010000:00020000:2.0:1767321087.943658:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321087.943667:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2786 00000100:00100000:2.0:1767321087.943693:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604582d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983234944:12345-0@lo:38:kworker.0 Request processed in 45us (135us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321087.943700:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2787 00000100:00080000:3.0:1767321087.943701:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321087.943708:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321087.943712:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983234944:0@lo:38:kworker.0 00000100:00080000:3.0:1767321087.943723:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321087.943727:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321087.943730:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983234432:0@lo:38:kworker.0 00000100:00100000:0.0:1767321087.943827:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983234560 00000100:00100000:0.0:1767321087.943832:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983234560 00000100:00100000:0.0:1767321087.943978:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 721 00000100:00100000:0.0:1767321087.943982:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983234560:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321087.943992:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321087.944032:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983234560:12345-0@lo:38:kworker.0 Request processed in 48us (544us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321087.944037:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 721 00000100:00080000:3.0:1767321087.944094:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321087.944098:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321087.944101:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463396d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983234560:0@lo:38:kworker.0 00010000:00010000:2.0:1767321088.133066:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1767321088.908783:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915718016 offset 0 length 520: 4 00000100:00100000:3.0:1767321088.908835:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321088.909456:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321088.928109:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718272 00000100:00100000:0.0:1767321088.928123:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718272 00000100:00100000:0.0:1767321088.928216:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718144 00000100:00100000:0.0:1767321088.928219:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718144 00000100:00100000:0.0:1767321088.928240:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2105 00000100:00100000:0.0:1767321088.928249:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915718272:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321088.928370:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915718272:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 122us (19534us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321088.928377:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2105 00000100:00100000:0.0:1767321088.928397:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2788 00000100:00100000:0.0:1767321088.928401:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915718144:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321088.928421:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321088.928448:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396fb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915718144:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (18995us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321088.928454:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2788 00000100:00080000:3.0:1767321088.929075:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321089.156696:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321090.180432:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321091.208257:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321092.037455:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321067/real 1767321067] req@ffff9346382dc380 x1853163983229056/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321092 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321092.037489:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346382dc380 x1853163983229056/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321092 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321092.037503:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.037527:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321092.037531:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983229056:0@lo:250:kworker.0 00000100:00080000:1.0:1767321092.037706:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321092.037712:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321092.037715:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321092.037720:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6315 00000100:00080000:1.0:1767321092.037724:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321092.037730:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321092.037759:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd180 x1853163983235072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321092.037806:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321092.037810:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321092.037811:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321092.037815:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6336 00000100:00080000:1.0:1767321092.037817:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321092.037820:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321092.037830:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163983235328/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321092.037848:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321092.037851:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321092.037852:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321092.037855:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6336 00000100:00080000:1.0:1767321092.037858:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321092.037860:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321092.037869:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100ff480 x1853163983235584/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321092.037876:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321092.037879:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321092.037881:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321092.037884:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6336 00000100:00080000:1.0:1767321092.037886:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321092.037888:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321092.037899:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd500 x1853163983235712/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321092.037915:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321092.037918:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321092.037920:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321092.037922:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6336 00000100:00080000:1.0:1767321092.037924:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321092.037927:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321092.037934:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fea00 x1853163983236096/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321092.038018:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983235968:0@lo:400:kworker.0 00000100:00100000:2.0:1767321092.038094:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.038148:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321092.038331:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983235200:0@lo:400:kworker.0 00000100:00100000:2.0:1767321092.038361:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.038396:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235200 00000100:00100000:2.0:1767321092.038403:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235200 00000100:00100000:2.0:1767321092.038623:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106 00000100:00100000:2.0:1767321092.038633:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983235200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321092.038683:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604581f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983235200:12345-0@lo:400:kworker.0 Request processed in 52us (321us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321092.038690:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106 00010000:00080000:2.0:1767321092.038726:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fed80 x1853163983235200/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321108 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321092.038738:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983235200:0@lo:400:kworker.0 00000100:00100000:1.0:1767321092.038808:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235968 00000100:00100000:1.0:1767321092.038812:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235968 00000100:00100000:1.0:1767321092.038853:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1291 00000100:00100000:1.0:1767321092.038860:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983235968:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321092.038896:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604583480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983235968:12345-0@lo:400:kworker.0 Request processed in 37us (802us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321092.038902:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1291 00000100:00100000:3.0:1767321092.038973:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983235456:0@lo:400:kworker.0 00000100:00100000:3.0:1767321092.039002:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321092.039030:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321092.039047:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983235840:0@lo:400:kworker.0 00000100:00100000:3.0:1767321092.039069:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321092.039092:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321092.039097:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321092.039105:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235840 00000100:00100000:3.0:1767321092.039109:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235840 00000100:00100000:3.0:1767321092.039137:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1292 00000100:00100000:3.0:1767321092.039144:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983235840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321092.039395:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983235840:12345-0@lo:400:kworker.0 Request processed in 251us (324us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321092.039403:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1292 00000100:00100000:3.0:1767321092.039433:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983235840:0@lo:400:kworker.0 00000100:00080000:3.0:1767321092.039448:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321092.039457:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983235072:0@lo:250:kworker.0 00000400:00000100:3.0:1767321092.039476:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983235072 offset 0 length 520: 4 00000100:00100000:3.0:1767321092.039494:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983235328:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039513:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321092.039531:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235456 00000100:00100000:3.0:1767321092.039534:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235456 00000100:00100000:3.0:1767321092.039558:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235328 00000100:00100000:3.0:1767321092.039561:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235328 00000100:00100000:3.0:1767321092.039575:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2107 00000100:00100000:3.0:1767321092.039581:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983235456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321092.039613:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983235456:12345-0@lo:400:kworker.0 Request processed in 31us (610us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321092.039619:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2107 00000100:00100000:3.0:1767321092.039630:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2789 00000100:00100000:3.0:1767321092.039635:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983235328:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321092.039651:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321092.039675:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463396c380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983235328:12345-0@lo:38:kworker.0 Request processed in 39us (161us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321092.039681:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2789 00010000:00080000:3.0:1767321092.039705:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe300 x1853163983235456/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321108 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321092.039714:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983235456:0@lo:400:kworker.0 00000100:00080000:3.0:1767321092.039727:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321092.039739:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.039744:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321092.039747:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983235328:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039754:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983235584:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039774:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321092.039791:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235584 00000100:00100000:3.0:1767321092.039794:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235584 00000100:00100000:3.0:1767321092.039810:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2790 00000100:00100000:3.0:1767321092.039815:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983235584:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321092.039824:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321092.039846:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983235584:12345-0@lo:38:kworker.0 Request processed in 30us (71us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321092.039852:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2790 00000100:00080000:3.0:1767321092.039872:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.039876:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321092.039879:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983235584:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039884:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983235712:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039909:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321092.039925:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983236096:0@lo:38:kworker.0 00000100:00100000:3.0:1767321092.039943:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321092.039958:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236096 00000100:00100000:3.0:1767321092.039961:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236096 00000100:00100000:3.0:1767321092.039977:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2791 00000100:00100000:3.0:1767321092.039981:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983236096:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321092.039991:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321092.040011:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983236096:12345-0@lo:38:kworker.0 Request processed in 29us (67us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321092.040017:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2791 00000100:00100000:0.0:1767321092.040411:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983235712 00000100:00100000:0.0:1767321092.040415:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983235712 00000100:00100000:0.0:1767321092.040442:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 722 00000100:00100000:0.0:1767321092.040447:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983235712:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321092.040459:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321092.040489:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983235712:12345-0@lo:38:kworker.0 Request processed in 40us (578us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321092.040495:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 722 00000100:00080000:3.0:1767321092.040582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.040587:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321092.040591:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983236096:0@lo:38:kworker.0 00000100:00080000:3.0:1767321092.040618:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321092.040622:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321092.040625:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983235712:0@lo:38:kworker.0 00000100:00100000:2.0:1767321092.040748:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983235968:0@lo:400:kworker.0 00010000:00010000:3.0:1767321092.228489:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321092.549626:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983236224:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321092.549712:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321092.549972:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321092.549994:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983236352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321092.550060:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321092.553656:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236224 00000100:00100000:2.0:1767321092.553675:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236224 00000100:00100000:3.0:1767321092.556340:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236352 00000100:00100000:3.0:1767321092.556364:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236352 00000100:00100000:3.0:1767321092.556754:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1404 00000100:00100000:3.0:1767321092.556770:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983236224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321092.556821:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321092.557026:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983236224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 260us (7313us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321092.557034:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1404 00000100:00100000:3.0:1767321092.557058:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1405 00000100:00100000:3.0:1767321092.557066:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983236352:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321092.557087:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321092.557132:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983236352:12345-0@lo:13:osp-pre-0-1.0 Request processed in 66us (7072us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321092.557138:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1405 00000100:00100000:3.0:1767321092.557224:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983236352:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321092.561639:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983236224:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321093.252477:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321094.028628:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321094.028740:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321094.028862:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718528 00000100:00100000:0.0:1767321094.028875:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718528 00000100:00080000:3.0:1767321094.028970:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321094.028984:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718400 00000100:00100000:0.0:1767321094.028989:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718400 00000100:00100000:0.0:1767321094.029023:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2108 00000100:00100000:0.0:1767321094.029036:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915718528:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321094.029108:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2792 00000100:00100000:3.0:1767321094.029117:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718400:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321094.029145:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321094.029162:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915718528:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 128us (551us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321094.029172:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2108 00000100:00100000:3.0:1767321094.029200:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718400:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 82us (458us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321094.029209:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2792 00010000:00010000:2.0:1767321094.278187:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321095.300729:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321096.324489:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321097.156700:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321097.156716:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321097.156789:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321097.156795:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321097.156800:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321097.156825:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6340 00000100:00100000:1.0:1767321097.156832:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983236480:0@lo:400:kworker.0 00000100:00080000:3.0:1767321097.156833:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321097.156845:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321097.156875:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a8380 x1853163983236608/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321097.156921:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321097.156925:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321097.156927:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321097.156931:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6340 00000100:00080000:3.0:1767321097.156934:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321097.156939:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321097.156950:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0aa300 x1853163983236864/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321097.156969:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321097.156972:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321097.156974:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321097.156977:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6340 00000100:00080000:3.0:1767321097.156979:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321097.156983:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321097.156999:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0ab100 x1853163983236992/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321097.157022:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321097.157026:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321097.157028:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321097.157031:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6340 00000100:00080000:3.0:1767321097.157034:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321097.157037:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321097.157049:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e0a9880 x1853163983237376/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321097.157081:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983236736:0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.157136:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321097.157192:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321097.157199:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983237120:0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.157229:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321097.157269:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236736 00000100:00100000:3.0:1767321097.157301:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236736 00000100:00100000:3.0:1767321097.157373:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109 00000100:00100000:3.0:1767321097.157384:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983236736:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.157441:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983236736:12345-0@lo:400:kworker.0 Request processed in 60us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321097.157449:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109 00000100:00100000:3.0:1767321097.157479:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237120 00000100:00100000:3.0:1767321097.157483:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237120 00000100:00100000:3.0:1767321097.157512:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1293 00000100:00100000:3.0:1767321097.157519:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983237120:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.157550:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0abb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983237120:12345-0@lo:400:kworker.0 Request processed in 31us (320us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321097.157556:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1293 00000100:00080000:3.0:1767321097.157576:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321097.157581:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321097.157615:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983237120:0@lo:400:kworker.0 00010000:00080000:3.0:1767321097.157627:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0aaa00 x1853163983236736/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321113 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321097.157635:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983236736:0@lo:400:kworker.0 00000100:00100000:1.0:1767321097.157787:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321097.157825:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321097.157834:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983237248:0@lo:400:kworker.0 00000100:00100000:1.0:1767321097.157863:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321097.158034:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236480 00000100:00100000:3.0:1767321097.158039:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236480 00000100:00100000:3.0:1767321097.158086:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2110 00000100:00100000:3.0:1767321097.158093:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983236480:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321097.158144:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983236480:12345-0@lo:400:kworker.0 Request processed in 52us (357us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321097.158154:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2110 00000100:00100000:2.0:1767321097.158311:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983236608:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.158348:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.158373:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983236864:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.158424:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.158448:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236608 00000100:00100000:2.0:1767321097.158452:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236608 00000100:00100000:2.0:1767321097.158473:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236864 00000100:00100000:2.0:1767321097.158476:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236864 00000100:00100000:2.0:1767321097.158495:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2793 00000100:00100000:2.0:1767321097.158501:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983236608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.158523:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.158553:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983236608:12345-0@lo:38:kworker.0 Request processed in 51us (204us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.158560:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2793 00000100:00100000:2.0:1767321097.158573:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2794 00000100:00100000:2.0:1767321097.158577:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983236864:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.158585:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.158606:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983236864:12345-0@lo:38:kworker.0 Request processed in 28us (182us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.158611:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2794 00000100:00080000:2.0:1767321097.158640:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321097.158646:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321097.158649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983236864:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.158657:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983236992:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.158683:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.158703:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983237376:0@lo:38:kworker.0 00000100:00100000:2.0:1767321097.158725:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321097.158742:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237376 00000100:00100000:2.0:1767321097.158745:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237376 00000100:00100000:2.0:1767321097.158763:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2795 00000100:00100000:2.0:1767321097.158767:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983237376:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.158776:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.158800:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983237376:12345-0@lo:38:kworker.0 Request processed in 32us (74us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.158806:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2795 00000100:00080000:3.0:1767321097.158944:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321097.158949:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321097.158952:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983237376:0@lo:38:kworker.0 00000100:00080000:3.0:1767321097.158988:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321097.158992:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321097.158995:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983236608:0@lo:38:kworker.0 00000100:00100000:0.0:1767321097.159175:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237248 00000100:00100000:0.0:1767321097.159255:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237248 00000100:00100000:2.0:1767321097.159319:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983236992 00000100:00100000:2.0:1767321097.159324:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983236992 00000100:00100000:2.0:1767321097.159349:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 723 00000100:00100000:2.0:1767321097.159355:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983236992:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321097.159366:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321097.159410:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983236992:12345-0@lo:38:kworker.0 Request processed in 54us (726us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321097.159418:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 723 00000100:00080000:2.0:1767321097.159444:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321097.159449:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321097.159452:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983236992:0@lo:38:kworker.0 00000100:00100000:0.0:1767321097.159519:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1294 00000100:00080000:1.0:1767321097.159526:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321097.159532:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983237248:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321097.159536:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321097.159574:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463e0a8a80 x1853163983236480/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321113 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321097.159587:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100ff480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983237248:12345-0@lo:400:kworker.0 Request processed in 59us (1722us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321097.159589:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983236480:0@lo:400:kworker.0 00000100:00100000:0.0:1767321097.159602:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1294 00000100:00100000:1.0:1767321097.159619:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983237248:0@lo:400:kworker.0 00010000:00010000:2.0:1767321097.353204:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321097.668657:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321097.668700:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983237504:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321097.668700:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983237632:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321097.671872:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321097.672037:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321097.682221:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237504 00000100:00100000:0.0:1767321097.682244:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237504 00000100:00100000:0.0:1767321097.682413:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237632 00000100:00100000:0.0:1767321097.682419:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237632 00000100:00100000:3.0:1767321097.682487:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1406 00000100:00100000:3.0:1767321097.682511:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983237504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321097.682804:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321097.682938:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983237504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 439us (11076us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321097.682946:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1406 00000100:00100000:3.0:1767321097.682971:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1407 00000100:00100000:3.0:1767321097.682979:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983237632:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321097.682999:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321097.683045:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983237632:12345-0@lo:13:osp-pre-0-1.0 Request processed in 67us (11010us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321097.683052:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1407 00000100:00100000:3.0:1767321097.683129:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983237632:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321097.683707:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983237504:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321098.372579:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321099.150396:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321099.150707:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321099.150789:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718784 00000100:00100000:2.0:1767321099.150806:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718784 00000100:00080000:1.0:1767321099.150896:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321099.151111:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718656 00000100:00100000:2.0:1767321099.151121:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718656 00000100:00100000:2.0:1767321099.151161:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2111 00000100:00100000:2.0:1767321099.151174:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915718784:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321099.151329:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915718784:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 157us (943us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321099.151339:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2111 00000100:00100000:2.0:1767321099.151360:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2796 00000100:00100000:2.0:1767321099.151366:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718656:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321099.151392:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321099.151424:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718656:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (718us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321099.151430:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2796 00010000:00010000:2.0:1767321099.396723:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321100.420736:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321101.444706:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321102.276462:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321102.276479:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1767321102.276623:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767321102.276757:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321102.276763:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321102.276768:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321102.276778:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6345 00000100:00080000:1.0:1767321102.276784:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321102.276793:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321102.276824:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc700 x1853163983237888/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321102.276885:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321102.276889:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321102.276891:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321102.276894:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6345 00000100:00080000:1.0:1767321102.276897:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321102.276902:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321102.276914:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fd880 x1853163983238144/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321102.276923:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321102.276927:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321102.276928:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321102.276931:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6345 00000100:00080000:1.0:1767321102.276934:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321102.276938:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321102.276952:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463e6f3b80 x1853163983238272/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321102.277016:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983237888:0@lo:38:kworker.0 00000100:00100000:0.0:1767321102.277244:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983238016:0@lo:400:kworker.0 00000100:00100000:0.0:1767321102.277370:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321102.277422:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321102.277428:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983238400:0@lo:400:kworker.0 00000100:00100000:0.0:1767321102.277458:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321102.277496:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238016 00000100:00100000:0.0:1767321102.277516:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238016 00000100:00100000:0.0:1767321102.277590:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767321102.277600:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983238016:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321102.277661:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983238016:12345-0@lo:400:kworker.0 Request processed in 62us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321102.277670:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112 00000100:00100000:0.0:1767321102.277697:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238400 00000100:00100000:0.0:1767321102.277701:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238400 00000100:00100000:0.0:1767321102.277737:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1295 00000100:00100000:0.0:1767321102.277742:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983238400:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321102.277766:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f966a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983238400:12345-0@lo:400:kworker.0 Request processed in 24us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321102.277770:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1295 00000100:00080000:0.0:1767321102.277783:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321102.277787:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321102.277816:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983238400:0@lo:400:kworker.0 00010000:00080000:0.0:1767321102.277825:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fed80 x1853163983238016/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321118 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321102.277837:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983238016:0@lo:400:kworker.0 00000100:00100000:3.0:1767321102.278110:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.278131:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983238144:0@lo:38:kworker.0 00000100:00100000:3.0:1767321102.278147:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.278161:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237888 00000100:00100000:3.0:1767321102.278164:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237888 00000100:00100000:3.0:1767321102.278179:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238144 00000100:00100000:3.0:1767321102.278181:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238144 00000100:00100000:3.0:1767321102.278195:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2797 00000100:00100000:3.0:1767321102.278200:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983237888:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321102.278220:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321102.278249:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983237888:12345-0@lo:38:kworker.0 Request processed in 48us (140us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321102.278254:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2797 00000100:00100000:3.0:1767321102.278264:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2798 00000100:00100000:3.0:1767321102.278266:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983238144:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321102.278289:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321102.278306:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983238144:12345-0@lo:38:kworker.0 Request processed in 39us (159us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321102.278310:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2798 00000100:00080000:3.0:1767321102.278338:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321102.278342:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321102.278345:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983238144:0@lo:38:kworker.0 00000100:00100000:3.0:1767321102.278350:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983238272:0@lo:38:kworker.0 00000100:00100000:3.0:1767321102.278370:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.278388:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238272 00000100:00100000:3.0:1767321102.278390:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238272 00000100:00100000:3.0:1767321102.278404:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 724 00000100:00100000:3.0:1767321102.278407:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983238272:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321102.278413:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321102.278429:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aaa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983238272:12345-0@lo:38:kworker.0 Request processed in 21us (58us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321102.278433:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 724 00000100:00080000:3.0:1767321102.278463:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321102.278467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321102.278470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e6f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983238272:0@lo:38:kworker.0 00000100:00080000:3.0:1767321102.278495:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321102.278498:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321102.278510:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983237888:0@lo:38:kworker.0 00000100:00100000:2.0:1767321102.278703:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983238528:0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.278732:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321102.278754:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983237760:0@lo:400:kworker.0 00000100:00100000:2.0:1767321102.278767:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321102.278827:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983237760 00000100:00100000:0.0:1767321102.278830:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983237760 00000100:00100000:0.0:1767321102.278868:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2113 00000100:00100000:0.0:1767321102.278873:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983237760:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321102.278911:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983237760:12345-0@lo:400:kworker.0 Request processed in 38us (142us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321102.278918:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2113 00000100:00080000:2.0:1767321102.278966:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767321102.278984:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe300 x1853163983237760/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321118 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321102.278993:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983237760:0@lo:400:kworker.0 00000100:00100000:3.0:1767321102.279059:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238528 00000100:00100000:3.0:1767321102.279064:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238528 00000100:00100000:3.0:1767321102.279112:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1296 00000100:00100000:3.0:1767321102.279119:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983238528:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321102.279155:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983238528:12345-0@lo:400:kworker.0 Request processed in 37us (421us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321102.279161:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1296 00000100:00080000:2.0:1767321102.279214:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321102.279233:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983238528:0@lo:400:kworker.0 00000100:00080000:1.0:1767321102.279915:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321102.279921:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321102.279924:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321102.279930:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6345 00000100:00080000:1.0:1767321102.279934:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321102.279940:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321102.279972:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163983238656/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321102.280027:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983238656:0@lo:38:kworker.0 00000100:00100000:2.0:1767321102.280082:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321102.280191:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238656 00000100:00100000:0.0:1767321102.280197:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238656 00000100:00100000:0.0:1767321102.280232:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2799 00000100:00100000:0.0:1767321102.280239:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983238656:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321102.280255:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321102.280803:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983238656:12345-0@lo:38:kworker.0 Request processed in 561us (719us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321102.280811:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2799 00000100:00080000:2.0:1767321102.280883:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321102.280888:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321102.280892:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983238656:0@lo:38:kworker.0 00010000:00010000:2.0:1767321102.476017:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321102.788763:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321102.788797:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983238784:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321102.789060:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321102.789229:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238784 00000100:00100000:3.0:1767321102.789241:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238784 00000100:00100000:3.0:1767321102.789482:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1408 00000100:00100000:3.0:1767321102.789494:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983238784:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321102.789538:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321102.789597:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983238784:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (538us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321102.789605:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1408 00000100:00100000:2.0:1767321102.790238:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983238784:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321102.790637:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321102.790654:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983238912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321102.790718:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321102.791997:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983238912 00000100:00100000:1.0:1767321102.792014:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983238912 00000100:00100000:1.0:1767321102.792100:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1409 00000100:00100000:1.0:1767321102.792111:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983238912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321102.792151:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321102.792246:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f967100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983238912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 136us (1527us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321102.792254:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1409 00000100:00100000:0.0:1767321102.792697:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983238912:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321103.494896:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321104.272771:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321104.272824:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321104.272924:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915718912 00000100:00100000:0.0:1767321104.272938:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915718912 00000100:00100000:0.0:1767321104.273032:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719040 00000100:00100000:0.0:1767321104.273036:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719040 00000100:00100000:0.0:1767321104.273100:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2114 00000100:00100000:0.0:1767321104.273111:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915719040:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321104.273215:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915719040:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 106us (390us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321104.273224:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2114 00000100:00100000:0.0:1767321104.273245:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2800 00000100:00100000:0.0:1767321104.273250:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718912:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321104.273327:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321104.273356:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915718912:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 104us (600us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321104.273364:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2800 00000100:00080000:3.0:1767321104.273910:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321104.520628:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321105.544078:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321106.564692:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321107.396663:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321107.396677:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321107.396761:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321107.396766:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321107.396774:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321107.396785:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6350 00000100:00080000:1.0:1767321107.396791:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321107.396803:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321107.396833:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604320e00 x1853163983239168/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321107.396925:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321107.396929:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321107.396941:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321107.396946:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6350 00000100:00080000:1.0:1767321107.396949:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321107.396953:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321107.396969:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626680 x1853163983239424/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321107.396981:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321107.396984:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321107.396986:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321107.396990:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6350 00000100:00080000:1.0:1767321107.396992:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321107.396996:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321107.397009:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624e00 x1853163983239552/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321107.397034:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321107.397038:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321107.397040:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321107.397043:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6350 00000100:00080000:1.0:1767321107.397049:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321107.397054:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321107.397064:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625180 x1853163983239936/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321107.397158:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983239296:0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.397257:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.397352:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321107.397356:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983239808:0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.397359:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983239680:0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.397401:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.397442:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983239168:0@lo:38:kworker.0 00000100:00100000:1.0:1767321107.397513:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239296 00000100:00100000:1.0:1767321107.397521:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239296 00000100:00100000:1.0:1767321107.397579:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2115 00000100:00100000:1.0:1767321107.397588:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983239296:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321107.398350:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983239040:0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.402421:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.402517:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983239424:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.402575:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.402604:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983239552:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.402630:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.402647:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983239936:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.402661:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.402756:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239168 00000100:00100000:2.0:1767321107.402766:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239168 00000100:00100000:2.0:1767321107.402798:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239424 00000100:00100000:2.0:1767321107.402800:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239424 00000100:00100000:2.0:1767321107.402806:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239936 00000100:00100000:2.0:1767321107.402808:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239936 00000100:00100000:2.0:1767321107.402820:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2801 00000100:00100000:2.0:1767321107.402827:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239168:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.402853:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.402900:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239168:12345-0@lo:38:kworker.0 Request processed in 72us (493us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.402907:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2801 00000100:00100000:2.0:1767321107.402925:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2802 00000100:00100000:2.0:1767321107.402928:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239424:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.402947:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.402970:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239424:12345-0@lo:38:kworker.0 Request processed in 41us (396us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.402977:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2802 00000100:00100000:2.0:1767321107.402987:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2803 00000100:00100000:2.0:1767321107.402991:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239936:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.403003:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.403024:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983239936:12345-0@lo:38:kworker.0 Request processed in 31us (361us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.403029:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2803 00000100:00080000:2.0:1767321107.403078:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321107.403084:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321107.403088:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604320e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983239168:0@lo:38:kworker.0 00000100:00080000:2.0:1767321107.403097:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321107.403100:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321107.403103:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983239424:0@lo:38:kworker.0 00000100:00080000:2.0:1767321107.403109:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321107.403112:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321107.403114:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983239936:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.403153:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239552 00000100:00100000:2.0:1767321107.403156:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239552 00000100:00100000:2.0:1767321107.403489:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 725 00000100:00100000:2.0:1767321107.403494:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983239552:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321107.403505:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321107.403542:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983239552:12345-0@lo:38:kworker.0 Request processed in 46us (910us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321107.403548:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 725 00000100:00080000:2.0:1767321107.403574:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321107.403578:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321107.403581:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983239552:0@lo:38:kworker.0 00000100:00100000:2.0:1767321107.403637:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239680 00000100:00100000:2.0:1767321107.403641:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239680 00000100:00100000:2.0:1767321107.403684:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1297 00000100:00100000:2.0:1767321107.403690:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983239680:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321107.403742:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983239680:12345-0@lo:400:kworker.0 Request processed in 52us (6343us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321107.403748:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1297 00000100:00100000:2.0:1767321107.403788:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983239680:0@lo:400:kworker.0 00000100:00080000:2.0:1767321107.403836:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321107.403842:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321107.405898:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983239296:12345-0@lo:400:kworker.0 Request processed in 8295us (8625us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.405917:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2115 00000100:00100000:3.0:1767321107.406261:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321107.406387:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321107.406489:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239040 00000100:00100000:0.0:1767321107.406500:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239040 00000100:00080000:3.0:1767321107.406555:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321107.406574:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2116 00000100:00100000:0.0:1767321107.406589:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983239040:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321107.406671:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f965f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983239040:12345-0@lo:400:kworker.0 Request processed in 85us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321107.406677:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2116 00010000:00080000:3.0:1767321107.407987:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346333a1880 x1853163983239040/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321123 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321107.408008:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346333a1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983239040:0@lo:400:kworker.0 00010000:00080000:2.0:1767321107.408125:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625c00 x1853163983239296/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321123 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321107.408134:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983239296:0@lo:400:kworker.0 00000100:00100000:1.0:1767321107.408337:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983239808 00000100:00100000:1.0:1767321107.408354:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983239808 00000100:00100000:1.0:1767321107.409180:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1298 00000100:00100000:1.0:1767321107.409192:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983239808:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321107.409328:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983239808:12345-0@lo:400:kworker.0 Request processed in 139us (3076us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.409337:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1298 00000100:00100000:0.0:1767321107.410430:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983239808:0@lo:400:kworker.0 00000100:00080000:3.0:1767321107.412222:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:3.0:1767321107.590691:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321107.909144:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321107.909174:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983240064:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321107.909628:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321107.909834:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321107.909852:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983240192:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321107.909897:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321107.909946:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240064 00000100:00100000:1.0:1767321107.909959:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240064 00000100:00100000:0.0:1767321107.910197:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240192 00000100:00100000:0.0:1767321107.910202:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240192 00000100:00100000:0.0:1767321107.910253:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1410 00000100:00100000:0.0:1767321107.910264:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983240192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321107.910335:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321107.910638:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1411 00000100:00100000:1.0:1767321107.910646:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983240064:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321107.910665:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321107.910743:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983240064:12345-0@lo:13:osp-pre-0-1.0 Request processed in 98us (1118us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321107.910751:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1411 00000100:00100000:3.0:1767321107.910899:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983240064:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321107.911648:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f51500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983240192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1384us (1748us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321107.911657:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1410 00000100:00100000:2.0:1767321107.919609:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983240192:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321108.612945:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321109.396517:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321109.396612:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321109.396822:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719296 00000100:00100000:1.0:1767321109.396831:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719296 00000100:00100000:1.0:1767321109.396948:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719168 00000100:00100000:1.0:1767321109.396952:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719168 00000100:00100000:1.0:1767321109.396974:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2117 00000100:00100000:1.0:1767321109.396983:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915719296:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321109.397061:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915719296:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (555us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321109.397073:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2117 00000100:00100000:1.0:1767321109.397090:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2804 00000100:00100000:1.0:1767321109.397094:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915719168:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321109.397115:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321109.397147:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463f964000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915719168:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (535us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321109.397154:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2804 00000100:00080000:3.0:1767321109.397887:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321109.636575:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321110.660940:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321111.684549:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321112.518709:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321112.518719:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321112.518783:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321112.518787:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321112.518791:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321112.518799:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6355 00000100:00080000:0.0:1767321112.518804:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321112.518810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321112.518836:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fc700 x1853163983240448/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321112.518907:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321112.518910:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321112.518912:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321112.518916:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6355 00000100:00080000:0.0:1767321112.518918:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321112.518922:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321112.518933:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346100fed80 x1853163983240704/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321112.518942:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321112.518945:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321112.518947:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321112.518950:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6355 00000100:00080000:0.0:1767321112.518952:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321112.518955:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321112.518966:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934609af2300 x1853163983240832/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321112.518986:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321112.518989:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321112.518991:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321112.518994:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6355 00000100:00080000:0.0:1767321112.518996:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321112.518999:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321112.519009:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604374380 x1853163983241216/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321112.519074:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983241088:0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519137:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.519217:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321112.519224:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983240320:0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519250:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.519306:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983240960:0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519342:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.519360:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321112.519365:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983240576:0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519386:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321112.519762:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240320 00000100:00100000:1.0:1767321112.519769:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240320 00000100:00100000:1.0:1767321112.519818:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240576 00000100:00100000:1.0:1767321112.519822:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240576 00000100:00100000:1.0:1767321112.519857:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118 00000100:00100000:1.0:1767321112.519865:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983240320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519918:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983240320:12345-0@lo:400:kworker.0 Request processed in 53us (666us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321112.519929:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118 00000100:00100000:1.0:1767321112.519945:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2119 00000100:00100000:1.0:1767321112.519952:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983240576:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.519983:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983240576:12345-0@lo:400:kworker.0 Request processed in 32us (595us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321112.519988:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2119 00010000:00080000:1.0:1767321112.520034:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fd880 x1853163983240576/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321128 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321112.520048:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983240576:0@lo:400:kworker.0 00010000:00080000:1.0:1767321112.520079:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346100fe300 x1853163983240320/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321128 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321112.520087:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983240320:0@lo:400:kworker.0 00000100:00080000:3.0:1767321112.520451:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321112.520480:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983240448:0@lo:38:kworker.0 00000100:00100000:3.0:1767321112.520519:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.520554:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983240704:0@lo:38:kworker.0 00000100:00100000:3.0:1767321112.520574:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.520592:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983240832:0@lo:38:kworker.0 00000100:00100000:3.0:1767321112.520616:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.520636:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983241216:0@lo:38:kworker.0 00000100:00100000:3.0:1767321112.520655:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321112.520685:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240448 00000100:00100000:3.0:1767321112.520690:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240448 00000100:00100000:3.0:1767321112.520710:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240704 00000100:00100000:3.0:1767321112.520712:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240704 00000100:00100000:3.0:1767321112.520723:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241216 00000100:00100000:3.0:1767321112.520725:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241216 00000100:00100000:3.0:1767321112.520734:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2805 00000100:00100000:3.0:1767321112.520740:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983240448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321112.520758:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321112.520790:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983240448:12345-0@lo:38:kworker.0 Request processed in 49us (270us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321112.520796:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2805 00000100:00100000:3.0:1767321112.520809:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2806 00000100:00100000:3.0:1767321112.520813:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983240704:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321112.520821:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321112.520854:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983240704:12345-0@lo:38:kworker.0 Request processed in 40us (279us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321112.520860:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2806 00000100:00100000:3.0:1767321112.520872:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2807 00000100:00100000:3.0:1767321112.520876:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983241216:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321112.520885:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321112.520908:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983241216:12345-0@lo:38:kworker.0 Request processed in 30us (252us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321112.520914:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2807 00000100:00080000:3.0:1767321112.520942:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321112.520947:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321112.520951:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983240448:0@lo:38:kworker.0 00000100:00080000:3.0:1767321112.520958:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321112.520961:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321112.520963:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983240704:0@lo:38:kworker.0 00000100:00080000:3.0:1767321112.520969:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321112.520972:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321112.520975:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604374380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983241216:0@lo:38:kworker.0 00000100:00100000:0.0:1767321112.524920:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241088 00000100:00100000:0.0:1767321112.524938:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241088 00000100:00100000:0.0:1767321112.525038:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240960 00000100:00100000:0.0:1767321112.525042:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240960 00000100:00100000:0.0:1767321112.525076:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1299 00000100:00100000:0.0:1767321112.525087:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983241088:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321112.525189:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983241088:12345-0@lo:400:kworker.0 Request processed in 104us (6050us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321112.525197:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1299 00000100:00100000:0.0:1767321112.525215:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1300 00000100:00100000:0.0:1767321112.525221:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983240960:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321112.525252:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983240960:12345-0@lo:400:kworker.0 Request processed in 32us (5910us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321112.525259:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1300 00000100:00100000:2.0:1767321112.525473:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983240832 00000100:00100000:2.0:1767321112.525479:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983240832 00000100:00100000:2.0:1767321112.525511:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 726 00000100:00100000:2.0:1767321112.525518:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983240832:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321112.525536:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321112.525580:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983240832:12345-0@lo:38:kworker.0 Request processed in 62us (4963us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321112.525587:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 726 00000100:00080000:2.0:1767321112.525632:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321112.525637:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321112.525641:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609af2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983240832:0@lo:38:kworker.0 00000100:00100000:1.0:1767321112.526953:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934609a35880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983240960:0@lo:400:kworker.0 00000100:00100000:1.0:1767321112.526991:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604375180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983241088:0@lo:400:kworker.0 00000100:00080000:3.0:1767321112.527131:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321112.527137:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321112.708951:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321113.028535:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321113.028547:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321113.028565:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983241472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321113.028565:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983241344:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321113.029818:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321113.030030:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241344 00000100:00100000:0.0:1767321113.030039:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241344 00000100:00100000:0.0:1767321113.030098:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1412 00000100:00100000:0.0:1767321113.030108:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983241344:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321113.030139:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321113.030192:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983241344:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (379us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321113.030199:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1412 00000100:00100000:1.0:1767321113.030900:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321113.031184:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983241344:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321113.034014:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241472 00000100:00100000:3.0:1767321113.034030:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241472 00000100:00100000:3.0:1767321113.034106:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1413 00000100:00100000:3.0:1767321113.034117:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983241472:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321113.034151:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321113.034238:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983241472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 123us (3339us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321113.034245:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1413 00000100:00100000:1.0:1767321113.034408:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983241472:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321113.733043:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321114.509093:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321114.509268:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719680 00000100:00100000:2.0:1767321114.509317:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719680 00000100:00100000:2.0:1767321114.509405:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2120 00000100:00100000:2.0:1767321114.509415:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915719680:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321114.509502:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604376680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915719680:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 88us (421us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321114.509509:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2120 00000400:00000100:1.0:1767321114.512209:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915719424 offset 0 length 520: 4 00000100:00100000:1.0:1767321114.512257:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00080000:3.0:1767321114.512952:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321114.513778:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719552 00000100:00100000:2.0:1767321114.513809:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719552 00000100:00100000:2.0:1767321114.513875:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2808 00000100:00100000:2.0:1767321114.513885:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915719552:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321114.513909:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321114.513998:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915719552:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 112us (1740us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321114.514013:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2808 00010000:00010000:2.0:1767321114.758660:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321115.780533:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321116.804633:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321117.636567:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321092/real 1767321092] req@ffff9346100fd180 x1853163983235072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321117 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321117.636602:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346100fd180 x1853163983235072/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321117 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321117.636619:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.636644:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321117.636648:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346100fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983235072:0@lo:250:kworker.0 00000100:00080000:1.0:1767321117.636723:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.636729:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1767321117.636733:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.636737:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6340 00000100:00080000:1.0:1767321117.636742:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.636757:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321117.636791:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627100 x1853163983241600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321117.636838:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.636842:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.636844:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.636847:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.636850:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.636853:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321117.636870:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624380 x1853163983241856/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321117.636895:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.636898:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.636899:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.636902:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.636905:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.636908:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321117.636918:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627800 x1853163983242112/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321117.636926:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.636929:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.636931:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.636934:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.636936:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.636939:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321117.636950:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c626d80 x1853163983242240/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321117.636968:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321117.636971:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321117.636973:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321117.636976:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6361 00000100:00080000:1.0:1767321117.636978:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321117.636981:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321117.636992:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624a80 x1853163983242624/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321117.637015:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983241984:0@lo:400:kworker.0 00000100:00100000:1.0:1767321117.637079:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321117.637153:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321117.637160:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983242368:0@lo:400:kworker.0 00000100:00100000:1.0:1767321117.637185:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321117.637230:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242368 00000100:00100000:1.0:1767321117.637238:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242368 00000100:00100000:1.0:1767321117.637315:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1301 00000100:00100000:1.0:1767321117.637325:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983242368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321117.637377:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983242368:12345-0@lo:400:kworker.0 Request processed in 53us (189us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321117.637385:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1301 00000100:00080000:1.0:1767321117.637419:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321117.637453:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983242368:0@lo:400:kworker.0 00000100:00100000:2.0:1767321117.638048:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983241600:0@lo:250:kworker.0 00000400:00000100:2.0:1767321117.638081:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983241600 offset 0 length 520: 4 00000100:00100000:2.0:1767321117.638105:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983241856:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.638145:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321117.638173:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983242112:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.638204:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321117.638221:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983242240:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.638245:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321117.638261:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983242624:0@lo:38:kworker.0 00000100:00100000:2.0:1767321117.638302:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321117.638436:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241984 00000100:00100000:1.0:1767321117.638442:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241984 00000100:00100000:1.0:1767321117.638488:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241856 00000100:00100000:1.0:1767321117.638491:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241856 00000100:00100000:1.0:1767321117.638500:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242112 00000100:00100000:1.0:1767321117.638502:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242112 00000100:00100000:1.0:1767321117.638509:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242624 00000100:00100000:1.0:1767321117.638511:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242624 00000100:00100000:1.0:1767321117.638521:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121 00000100:00100000:1.0:1767321117.638540:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983241984:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321117.638587:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983241984:12345-0@lo:400:kworker.0 Request processed in 49us (1508us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321117.638595:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121 00000100:00100000:1.0:1767321117.638610:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2809 00000100:00100000:1.0:1767321117.638614:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983241856:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321117.638633:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321117.638662:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f50700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983241856:12345-0@lo:38:kworker.0 Request processed in 46us (516us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321117.638668:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2809 00000100:00100000:1.0:1767321117.638691:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1767321117.638696:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983242112:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321117.638705:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321117.638727:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f52a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983242112:12345-0@lo:38:kworker.0 Request processed in 30us (522us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321117.638733:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1767321117.638753:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2811 00000100:00100000:1.0:1767321117.638757:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983242624:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321117.638766:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321117.638791:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609af2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983242624:12345-0@lo:38:kworker.0 Request processed in 33us (489us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321117.638797:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2811 00010000:00080000:1.0:1767321117.638830:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624000 x1853163983241984/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321133 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321117.638842:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983241984:0@lo:400:kworker.0 00000100:00080000:2.0:1767321117.639225:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321117.639248:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.639253:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.639257:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983241856:0@lo:38:kworker.0 00000100:00080000:2.0:1767321117.639265:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.639289:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.639293:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983242112:0@lo:38:kworker.0 00000100:00080000:2.0:1767321117.639303:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321117.639307:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321117.639310:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983242624:0@lo:38:kworker.0 00000100:00100000:3.0:1767321117.640100:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983241728:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.640166:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.640206:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321117.640212:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983242496:0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.640234:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321117.640263:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983241728 00000100:00100000:3.0:1767321117.640288:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983241728 00000100:00100000:3.0:1767321117.640334:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2122 00000100:00100000:3.0:1767321117.640341:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983241728:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.640699:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0aa680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983241728:12345-0@lo:400:kworker.0 Request processed in 357us (531us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.640707:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2122 00000100:00100000:3.0:1767321117.640733:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242496 00000100:00100000:3.0:1767321117.640737:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242496 00000100:00100000:3.0:1767321117.640776:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1302 00000100:00100000:3.0:1767321117.640783:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983242496:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321117.640815:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a9f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983242496:12345-0@lo:400:kworker.0 Request processed in 32us (580us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321117.640821:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1302 00000100:00100000:3.0:1767321117.640851:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983242496:0@lo:400:kworker.0 00010000:00080000:3.0:1767321117.640863:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c625f80 x1853163983241728/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321133 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321117.640874:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983241728:0@lo:400:kworker.0 00000100:00100000:0.0:1767321117.641352:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242240 00000100:00100000:0.0:1767321117.641358:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242240 00000100:00100000:0.0:1767321117.641386:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 727 00000100:00100000:0.0:1767321117.641391:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983242240:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321117.641421:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321117.641463:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934610f53100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983242240:12345-0@lo:38:kworker.0 Request processed in 71us (3216us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321117.641469:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 727 00000100:00080000:0.0:1767321117.641502:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321117.641506:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321117.641509:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983242240:0@lo:38:kworker.0 00000100:00080000:2.0:1767321117.644478:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321117.644487:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321117.828514:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321118.148606:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983242752:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321118.148679:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321118.149010:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321118.149021:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983242880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321118.149073:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321118.149473:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242752 00000100:00100000:1.0:1767321118.149483:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242752 00000100:00100000:1.0:1767321118.149537:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983242880 00000100:00100000:1.0:1767321118.149542:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983242880 00000100:00100000:1.0:1767321118.149558:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1414 00000100:00100000:1.0:1767321118.149565:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983242752:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321118.149600:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321118.149659:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604320e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983242752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (981us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321118.149667:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1414 00000100:00100000:1.0:1767321118.149696:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1415 00000100:00100000:1.0:1767321118.149702:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983242880:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321118.149717:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321118.149762:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983242880:12345-0@lo:13:osp-pre-0-1.0 Request processed in 60us (688us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321118.149768:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1415 00000100:00100000:1.0:1767321118.149811:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983242880:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321118.150405:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983242752:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321118.853347:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321119.630239:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321119.630366:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321119.634816:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719936 00000100:00100000:1.0:1767321119.634834:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719936 00000100:00100000:1.0:1767321119.634933:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915719808 00000100:00100000:1.0:1767321119.634937:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915719808 00000100:00100000:1.0:1767321119.634968:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2123 00000100:00100000:1.0:1767321119.634979:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915719936:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767321119.635001:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321119.635068:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604375180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915719936:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 91us (4840us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321119.635073:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2812 00000100:00100000:1.0:1767321119.635078:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2123 00000100:00100000:0.0:1767321119.635080:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915719808:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321119.635103:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321119.635133:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604374380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915719808:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 53us (4768us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321119.635140:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2812 00010000:00010000:3.0:1767321119.879812:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321120.900556:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321121.925059:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321122.758727:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321122.758736:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321122.758798:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321122.758801:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321122.758805:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321122.758812:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6366 00000100:00080000:2.0:1767321122.758817:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321122.758823:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321122.758849:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1500 x1853163983243136/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321122.758903:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321122.758906:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321122.758908:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321122.758911:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6366 00000100:00080000:2.0:1767321122.758914:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321122.758917:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321122.758926:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163983243392/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321122.758934:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321122.758937:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321122.758939:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321122.758942:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6366 00000100:00080000:2.0:1767321122.758944:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321122.758947:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321122.758955:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3100 x1853163983243520/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321122.758971:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321122.758974:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321122.758976:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321122.758979:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6366 00000100:00080000:2.0:1767321122.758981:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321122.758984:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321122.758991:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163983243904/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321122.759024:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983243648:0@lo:400:kworker.0 00000100:00100000:2.0:1767321122.759089:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.759138:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321122.759144:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983243264:0@lo:400:kworker.0 00000100:00100000:2.0:1767321122.759168:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.759199:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243264 00000100:00100000:2.0:1767321122.759206:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243264 00000100:00100000:2.0:1767321122.759260:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124 00000100:00100000:2.0:1767321122.759314:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983243264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321122.759940:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983243008:0@lo:400:kworker.0 00000100:00100000:1.0:1767321122.759972:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321122.759993:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321122.759998:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983243776:0@lo:400:kworker.0 00000100:00100000:1.0:1767321122.760018:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321122.760044:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243008 00000100:00100000:1.0:1767321122.760049:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243008 00000100:00100000:1.0:1767321122.760080:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2125 00000100:00100000:1.0:1767321122.760087:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983243008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321122.760143:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983243008:12345-0@lo:400:kworker.0 Request processed in 56us (168us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321122.760151:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2125 00010000:00080000:1.0:1767321122.760194:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3480 x1853163983243008/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321138 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321122.760207:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983243008:0@lo:400:kworker.0 00000100:00100000:3.0:1767321122.761087:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243648 00000100:00100000:3.0:1767321122.761183:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243648 00000100:00100000:3.0:1767321122.761237:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243776 00000100:00100000:3.0:1767321122.761241:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243776 00000100:00100000:2.0:1767321122.761498:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983243264:12345-0@lo:400:kworker.0 Request processed in 2228us (2326us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321122.761516:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124 00010000:00080000:2.0:1767321122.761561:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0700 x1853163983243264/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321138 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321122.761576:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983243264:0@lo:400:kworker.0 00000100:00080000:2.0:1767321122.761614:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321122.761619:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321122.761622:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321122.761640:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983243136:0@lo:38:kworker.0 00000100:00100000:2.0:1767321122.761688:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.761709:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983243392:0@lo:38:kworker.0 00000100:00100000:2.0:1767321122.761731:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.761748:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983243520:0@lo:38:kworker.0 00000100:00100000:2.0:1767321122.761771:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.761789:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983243904:0@lo:38:kworker.0 00000100:00100000:2.0:1767321122.761807:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321122.761834:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243136 00000100:00100000:2.0:1767321122.761838:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243136 00000100:00100000:2.0:1767321122.761872:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243392 00000100:00100000:2.0:1767321122.761875:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243392 00000100:00100000:2.0:1767321122.761898:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243904 00000100:00100000:2.0:1767321122.761901:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243904 00000100:00100000:2.0:1767321122.761912:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2813 00000100:00100000:2.0:1767321122.761917:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243136:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321122.761939:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321122.761975:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243136:12345-0@lo:38:kworker.0 Request processed in 57us (287us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321122.761982:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2813 00000100:00100000:2.0:1767321122.761993:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2814 00000100:00100000:2.0:1767321122.761997:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243392:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321122.762006:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321122.762030:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243392:12345-0@lo:38:kworker.0 Request processed in 32us (298us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321122.762035:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2814 00000100:00100000:2.0:1767321122.762046:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2815 00000100:00100000:2.0:1767321122.762049:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243904:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321122.762058:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321122.762079:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983243904:12345-0@lo:38:kworker.0 Request processed in 28us (271us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321122.762084:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2815 00000100:00080000:2.0:1767321122.762127:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321122.762131:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321122.762135:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983243136:0@lo:38:kworker.0 00000100:00080000:2.0:1767321122.762143:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321122.762145:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321122.762148:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983243392:0@lo:38:kworker.0 00000100:00080000:2.0:1767321122.762154:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321122.762157:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321122.762159:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983243904:0@lo:38:kworker.0 00000100:00100000:3.0:1767321122.762328:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1303 00000100:00100000:3.0:1767321122.762339:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983243648:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321122.762400:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983243648:12345-0@lo:400:kworker.0 Request processed in 64us (3309us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321122.762406:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1303 00000100:00100000:3.0:1767321122.762419:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1304 00000100:00100000:3.0:1767321122.762426:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983243776:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321122.762461:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983243776:12345-0@lo:400:kworker.0 Request processed in 36us (2442us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321122.762467:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1304 00000100:00100000:1.0:1767321122.763311:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983243776:0@lo:400:kworker.0 00000100:00100000:1.0:1767321122.763381:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983243520 00000100:00100000:1.0:1767321122.763387:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983243520 00000100:00100000:1.0:1767321122.763417:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 728 00000100:00100000:1.0:1767321122.763421:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983243520:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321122.763435:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321122.763493:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983243520:12345-0@lo:38:kworker.0 Request processed in 69us (1720us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321122.763499:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 728 00000100:00100000:1.0:1767321122.763566:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983243648:0@lo:400:kworker.0 00000100:00080000:0.0:1767321122.764058:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321122.764065:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321122.764068:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983243520:0@lo:38:kworker.0 00010000:00010000:3.0:1767321122.950793:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321123.269050:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321123.269071:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983244032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321123.269166:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321123.269379:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983244160:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321123.269407:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321123.269471:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244032 00000100:00100000:3.0:1767321123.269478:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244032 00000100:00100000:3.0:1767321123.269521:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244160 00000100:00100000:3.0:1767321123.269524:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244160 00000100:00100000:3.0:1767321123.269541:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1416 00000100:00100000:3.0:1767321123.269549:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983244032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321123.269580:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321123.269637:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983244032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (470us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321123.269654:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1416 00000100:00100000:3.0:1767321123.269671:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1417 00000100:00100000:3.0:1767321123.269677:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983244160:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321123.269699:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321123.269738:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983244160:12345-0@lo:13:osp-pre-0-1.0 Request processed in 61us (329us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321123.269743:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1417 00000100:00100000:3.0:1767321123.269782:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983244160:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321123.269812:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983244032:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321123.972509:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321124.749145:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321124.749196:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321124.753906:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720064 00000100:00100000:0.0:1767321124.753923:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720064 00000100:00100000:0.0:1767321124.753979:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720192 00000100:00100000:0.0:1767321124.753983:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720192 00000100:00100000:0.0:1767321124.754045:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2126 00000100:00100000:0.0:1767321124.754055:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915720192:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321124.754139:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915720192:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (4942us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321124.754146:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2126 00000100:00100000:0.0:1767321124.754165:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2816 00000100:00100000:0.0:1767321124.754170:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915720064:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321124.754190:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321124.754218:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0a8a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915720064:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 46us (5083us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321124.754224:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2816 00000100:00080000:1.0:1767321124.754698:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321124.996771:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321126.020541:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321127.048711:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321127.876351:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321127.876361:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767321127.876459:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983244288:0@lo:400:kworker.0 00000100:00100000:1.0:1767321127.876543:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321127.876660:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244288 00000100:00100000:3.0:1767321127.876669:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244288 00000100:00100000:3.0:1767321127.876743:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2127 00000100:00100000:3.0:1767321127.876753:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983244288:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321127.876878:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321127.877093:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321127.877098:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321127.877102:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321127.877108:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6371 00000100:00080000:2.0:1767321127.877112:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321127.877118:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321127.877140:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f2a00 x1853163983244416/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321127.877179:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321127.877183:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321127.877185:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321127.877189:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6371 00000100:00080000:2.0:1767321127.877192:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321127.877195:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321127.877207:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3b80 x1853163983244672/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321127.877215:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321127.877218:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321127.877220:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321127.877223:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6371 00000100:00080000:2.0:1767321127.877225:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321127.877229:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321127.877240:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163983244800/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321127.877301:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983245056:0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877338:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.877387:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245056 00000100:00100000:0.0:1767321127.877392:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245056 00000100:00100000:0.0:1767321127.877431:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1305 00000100:00100000:0.0:1767321127.877438:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983245056:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877489:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983245056:12345-0@lo:400:kworker.0 Request processed in 51us (150us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321127.877496:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1305 00000100:00100000:0.0:1767321127.877529:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983244544:0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877564:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.877584:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321127.877589:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983244928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877609:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.877633:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244544 00000100:00100000:0.0:1767321127.877637:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244544 00000100:00100000:0.0:1767321127.877664:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2128 00000100:00100000:0.0:1767321127.877670:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983244544:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877706:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983244544:12345-0@lo:400:kworker.0 Request processed in 35us (150us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321127.877712:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2128 00000100:00080000:0.0:1767321127.877730:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:0.0:1767321127.877733:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321127.877758:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983245056:0@lo:400:kworker.0 00010000:00080000:0.0:1767321127.877786:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1500 x1853163983244544/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321143 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321127.877796:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983244544:0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877809:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244928 00000100:00100000:0.0:1767321127.877812:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244928 00000100:00100000:0.0:1767321127.877838:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1306 00000100:00100000:0.0:1767321127.877844:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983244928:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877875:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983244928:12345-0@lo:400:kworker.0 Request processed in 31us (264us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321127.877881:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1306 00000100:00080000:0.0:1767321127.877896:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321127.877909:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983244928:0@lo:400:kworker.0 00000100:00100000:0.0:1767321127.877938:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983244416:0@lo:38:kworker.0 00000100:00100000:0.0:1767321127.877959:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.877972:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983244672:0@lo:38:kworker.0 00000100:00100000:0.0:1767321127.877987:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.877996:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983244800:0@lo:38:kworker.0 00000100:00100000:0.0:1767321127.878013:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321127.878042:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244416 00000100:00100000:0.0:1767321127.878045:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244416 00000100:00100000:0.0:1767321127.878057:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244672 00000100:00100000:0.0:1767321127.878059:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244672 00000100:00100000:0.0:1767321127.878072:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2817 00000100:00100000:0.0:1767321127.878076:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983244416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321127.878093:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321127.878125:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983244416:12345-0@lo:38:kworker.0 Request processed in 47us (165us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321127.878131:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2817 00000100:00100000:0.0:1767321127.878142:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2818 00000100:00100000:0.0:1767321127.878145:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983244672:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321127.878153:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321127.878174:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983244672:12345-0@lo:38:kworker.0 Request processed in 27us (185us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321127.878179:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2818 00000100:00080000:0.0:1767321127.878202:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321127.878207:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321127.878210:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983244416:0@lo:38:kworker.0 00000100:00080000:0.0:1767321127.878218:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321127.878221:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321127.878223:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983244672:0@lo:38:kworker.0 00000100:00100000:0.0:1767321127.878254:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983244800 00000100:00100000:0.0:1767321127.878257:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983244800 00000100:00100000:0.0:1767321127.878305:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 729 00000100:00100000:0.0:1767321127.878309:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983244800:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321127.878317:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321127.878346:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983244800:12345-0@lo:38:kworker.0 Request processed in 35us (331us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321127.878351:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 729 00000100:00080000:0.0:1767321127.878372:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321127.878376:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321127.878379:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983244800:0@lo:38:kworker.0 00000100:00100000:3.0:1767321127.878854:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983244288:12345-0@lo:400:kworker.0 Request processed in 2101us (2308us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321127.878862:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2127 00000100:00080000:2.0:1767321127.878996:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321127.879000:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321127.879003:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321127.879007:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6371 00000100:00080000:2.0:1767321127.879010:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321127.879014:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321127.879037:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1180 x1853163983245184/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321127.879119:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983245184:0@lo:38:kworker.0 00000100:00100000:0.0:1767321127.879153:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321127.879227:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245184 00000100:00100000:3.0:1767321127.879232:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245184 00000100:00100000:3.0:1767321127.879258:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2819 00000100:00100000:3.0:1767321127.879263:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983245184:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321127.879287:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321127.879329:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983245184:12345-0@lo:38:kworker.0 Request processed in 65us (176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321127.879335:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2819 00000100:00080000:2.0:1767321127.879401:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321127.879406:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321127.879409:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983245184:0@lo:38:kworker.0 00010000:00080000:1.0:1767321127.879655:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0000 x1853163983244288/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321143 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321127.879667:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983244288:0@lo:400:kworker.0 00010000:00010000:2.0:1767321128.069295:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321128.388668:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983245312:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321128.388740:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321128.388921:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245312 00000100:00100000:3.0:1767321128.388930:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245312 00000100:00100000:3.0:1767321128.388988:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1418 00000100:00100000:3.0:1767321128.388997:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983245312:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321128.389028:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321128.389072:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983245312:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (332us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321128.389080:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1418 00000100:00100000:2.0:1767321128.389455:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463e0ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983245312:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321128.389726:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983245440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321128.389760:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321128.389869:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245440 00000100:00100000:2.0:1767321128.389875:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245440 00000100:00100000:2.0:1767321128.389913:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1419 00000100:00100000:2.0:1767321128.389920:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983245440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321128.389940:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321128.390228:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983245440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 307us (466us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321128.390236:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1419 00000100:00100000:0.0:1767321128.390324:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983245440:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321129.094902:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321129.872403:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321129.872451:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321129.878056:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720320 00000100:00100000:1.0:1767321129.878074:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720320 00000100:00100000:1.0:1767321129.878214:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720448 00000100:00100000:1.0:1767321129.878218:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720448 00000100:00100000:1.0:1767321129.878306:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2129 00000100:00100000:1.0:1767321129.878318:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915720448:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321129.878404:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346333a1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915720448:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (5952us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321129.878412:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2129 00000100:00100000:1.0:1767321129.878432:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2820 00000100:00100000:1.0:1767321129.878448:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915720320:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321129.878471:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321129.878500:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e0ab480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915720320:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 51us (6112us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321129.879249:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2820 00000100:00080000:3.0:1767321129.880051:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321130.116659:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321131.141017:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321132.165172:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321132.996577:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321132.996586:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767321132.996699:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983245568:0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.996774:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321132.997424:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321132.997430:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321132.997443:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321132.997448:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6376 00000100:00080000:0.0:1767321132.997452:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321132.997468:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321132.997493:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e56a00 x1853163983245696/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321132.997530:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321132.997534:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321132.997536:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321132.997539:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6376 00000100:00080000:0.0:1767321132.997542:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321132.997546:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321132.997557:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55f80 x1853163983245952/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321132.997565:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321132.997568:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321132.997570:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321132.997572:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6376 00000100:00080000:0.0:1767321132.997574:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321132.997577:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321132.997589:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54380 x1853163983246080/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321132.997963:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983245696:0@lo:38:kworker.0 00000100:00100000:3.0:1767321132.998009:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321132.998341:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1767321132.998351:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983246336:0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998382:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321132.998426:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983245824:0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998475:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246336 00000100:00100000:1.0:1767321132.998482:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246336 00000100:00100000:1.0:1767321132.998535:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1307 00000100:00100000:1.0:1767321132.998544:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983246336:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998596:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983246336:12345-0@lo:400:kworker.0 Request processed in 52us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321132.998602:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1307 00000100:00100000:1.0:1767321132.998655:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983246336:0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998707:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983246208:0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998730:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321132.998755:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246208 00000100:00100000:1.0:1767321132.998759:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246208 00000100:00100000:1.0:1767321132.998781:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1308 00000100:00100000:1.0:1767321132.998787:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983246208:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321132.998833:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983246208:12345-0@lo:400:kworker.0 Request processed in 46us (102us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321132.998840:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1308 00000100:00100000:1.0:1767321132.999145:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983246208:0@lo:400:kworker.0 00000100:00080000:1.0:1767321132.999172:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321132.999176:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321132.999468:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983245952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321132.999497:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321133.001284:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245568 00000100:00100000:2.0:1767321133.001297:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245568 00000100:00100000:3.0:1767321133.001336:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983246080:0@lo:38:kworker.0 00000100:00100000:2.0:1767321133.001355:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245696 00000100:00100000:2.0:1767321133.001358:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245696 00000100:00100000:2.0:1767321133.001371:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245952 00000100:00100000:2.0:1767321133.001373:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245952 00000100:00100000:2.0:1767321133.001384:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130 00000100:00100000:3.0:1767321133.001384:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321133.001393:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983245568:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321133.001430:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2821 00000100:00100000:2.0:1767321133.001439:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983245568:12345-0@lo:400:kworker.0 Request processed in 48us (4665us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321133.001446:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130 00000100:00100000:2.0:1767321133.001473:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2822 00000100:00100000:2.0:1767321133.001478:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983245952:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321133.001497:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321133.001531:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983245952:12345-0@lo:38:kworker.0 Request processed in 52us (2032us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321133.001537:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2822 00000100:00100000:3.0:1767321133.001552:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983245696:12345-0@lo:38:kworker.0 00000100:00080000:2.0:1767321133.001554:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:3.0:1767321133.001563:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321133.001591:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463e6f3b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983245696:12345-0@lo:38:kworker.0 Request processed in 154us (3581us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321133.001596:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2821 00000100:00080000:3.0:1767321133.001626:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321133.001631:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321133.001634:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983245696:0@lo:38:kworker.0 00000100:00080000:3.0:1767321133.001643:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321133.001647:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321133.001649:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983245952:0@lo:38:kworker.0 00000100:00100000:3.0:1767321133.001679:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246080 00000100:00100000:3.0:1767321133.001682:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246080 00000100:00100000:3.0:1767321133.001703:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 730 00000100:00100000:3.0:1767321133.001707:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983246080:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321133.001716:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321133.001744:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983246080:12345-0@lo:38:kworker.0 Request processed in 36us (360us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321133.001749:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 730 00000100:00080000:3.0:1767321133.001771:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321133.001775:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321133.001778:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983246080:0@lo:38:kworker.0 00000100:00100000:3.0:1767321133.001812:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321133.001927:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983245824 00000100:00100000:3.0:1767321133.001931:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983245824 00000100:00100000:3.0:1767321133.001958:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2131 00000100:00100000:3.0:1767321133.001963:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983245824:12345-0@lo:400:kworker.0 00010000:00080000:2.0:1767321133.002033:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54700 x1853163983245568/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321148 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321133.002045:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983245568:0@lo:400:kworker.0 00000100:00080000:2.0:1767321133.002060:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321133.002063:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983245824:12345-0@lo:400:kworker.0 Request processed in 100us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321133.002070:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2131 00010000:00080000:2.0:1767321133.002087:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e57100 x1853163983245824/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321148 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321133.002095:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983245824:0@lo:400:kworker.0 00000100:00080000:0.0:1767321133.002150:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321133.002155:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321133.002158:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321133.002162:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6376 00000100:00080000:0.0:1767321133.002166:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321133.002170:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321133.002197:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55c00 x1853163983246464/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321133.002280:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983246464:0@lo:38:kworker.0 00000100:00100000:3.0:1767321133.002300:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321133.002337:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246464 00000100:00100000:1.0:1767321133.002342:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246464 00000100:00100000:1.0:1767321133.002410:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2823 00000100:00100000:1.0:1767321133.002415:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983246464:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321133.002425:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321133.002645:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983246464:12345-0@lo:38:kworker.0 Request processed in 227us (342us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321133.002652:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2823 00000100:00080000:0.0:1767321133.002706:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321133.002710:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321133.002714:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983246464:0@lo:38:kworker.0 00010000:00010000:2.0:1767321133.190398:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321133.508830:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983246592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321133.508918:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321133.509055:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246592 00000100:00100000:3.0:1767321133.509065:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246592 00000100:00100000:3.0:1767321133.509128:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1420 00000100:00100000:3.0:1767321133.509139:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983246592:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321133.509181:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321133.509240:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983246592:12345-0@lo:13:osp-pre-0-1.0 Request processed in 92us (310us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321133.509249:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1420 00000100:00100000:2.0:1767321133.509788:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983246592:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321133.509943:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983246720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321133.509979:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321133.510067:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246720 00000100:00100000:0.0:1767321133.510073:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246720 00000100:00100000:0.0:1767321133.510113:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1421 00000100:00100000:0.0:1767321133.510120:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983246720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321133.510140:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321133.510267:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983246720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 147us (287us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321133.510295:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1421 00000100:00100000:1.0:1767321133.510435:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983246720:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321134.214056:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321134.992263:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321134.992421:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321134.992594:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720704 00000100:00100000:3.0:1767321134.992604:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720704 00000100:00100000:0.0:1767321134.992668:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720576 00000100:00100000:0.0:1767321134.992672:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720576 00000100:00100000:3.0:1767321134.992687:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2132 00000100:00100000:3.0:1767321134.992697:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915720704:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321134.992698:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2824 00000100:00100000:0.0:1767321134.992702:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915720576:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321134.992722:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321134.992985:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321134.993062:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915720704:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 365us (808us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321134.993070:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2132 00000100:00100000:0.0:1767321135.001694:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915720576:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 8978us (9261us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321135.001710:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2824 00010000:00010000:3.0:1767321135.236528:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321136.260744:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321137.284556:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321138.117155:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321138.117166:0:54833:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767321138.117292:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983246848:0@lo:400:kworker.0 00000100:00100000:3.0:1767321138.117383:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.117502:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246848 00000100:00100000:0.0:1767321138.117511:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246848 00000100:00100000:0.0:1767321138.117582:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133 00000100:00100000:0.0:1767321138.117593:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983246848:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321138.117681:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321138.119599:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321138.119610:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321138.119615:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321138.119622:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6381 00000100:00080000:2.0:1767321138.119627:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321138.119634:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321138.119672:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1500 x1853163983246976/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321138.119729:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321138.119733:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321138.119735:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321138.119746:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6381 00000100:00080000:2.0:1767321138.119748:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321138.119752:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321138.119764:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3100 x1853163983247232/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321138.119773:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321138.119777:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321138.119778:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321138.119781:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6381 00000100:00080000:2.0:1767321138.119784:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321138.119787:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321138.119801:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163983247360/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321138.119893:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983247616:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.119952:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120033:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247616 00000100:00100000:1.0:1767321138.120040:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247616 00000100:00100000:1.0:1767321138.120085:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1309 00000100:00100000:1.0:1767321138.120092:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983247616:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120138:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983247616:12345-0@lo:400:kworker.0 Request processed in 46us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321138.120148:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1309 00000100:00080000:1.0:1767321138.120171:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321138.120200:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983247616:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120235:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983247104:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120263:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120306:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321138.120313:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983247488:0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120335:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120360:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983246976:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.120398:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120422:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983247232:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.120457:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120474:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983247360:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.120495:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321138.120534:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247104 00000100:00100000:1.0:1767321138.120538:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247104 00000100:00100000:1.0:1767321138.120569:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983246976 00000100:00100000:1.0:1767321138.120572:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983246976 00000100:00100000:1.0:1767321138.120581:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247232 00000100:00100000:1.0:1767321138.120584:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247232 00000100:00100000:1.0:1767321138.120593:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2134 00000100:00100000:1.0:1767321138.120599:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983247104:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120636:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934609a35880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983247104:12345-0@lo:400:kworker.0 Request processed in 37us (372us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321138.120642:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2134 00000100:00100000:1.0:1767321138.120655:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2825 00000100:00100000:1.0:1767321138.120660:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983246976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321138.120678:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321138.120709:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983246976:12345-0@lo:38:kworker.0 Request processed in 47us (309us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321138.120716:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2825 00000100:00100000:1.0:1767321138.120727:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2826 00000100:00100000:1.0:1767321138.120730:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983247232:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321138.120739:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321138.120763:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983247232:12345-0@lo:38:kworker.0 Request processed in 31us (305us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321138.120768:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2826 00000100:00100000:1.0:1767321138.120784:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247488 00000100:00100000:1.0:1767321138.120787:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247488 00000100:00100000:1.0:1767321138.120808:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1310 00000100:00100000:1.0:1767321138.120814:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983247488:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321138.120843:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983247488:12345-0@lo:400:kworker.0 Request processed in 30us (507us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321138.120849:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1310 00000100:00080000:1.0:1767321138.120864:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321138.120867:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321138.120885:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983247488:0@lo:400:kworker.0 00010000:00080000:1.0:1767321138.120894:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1880 x1853163983247104/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321138.120903:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983247104:0@lo:400:kworker.0 00000100:00080000:1.0:1767321138.120931:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.120935:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.120938:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983246976:0@lo:38:kworker.0 00000100:00080000:1.0:1767321138.120945:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.120948:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.120951:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983247232:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.121006:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247360 00000100:00100000:1.0:1767321138.121010:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247360 00000100:00100000:1.0:1767321138.121031:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 731 00000100:00100000:1.0:1767321138.121036:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983247360:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321138.121046:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321138.121073:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fe300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983247360:12345-0@lo:38:kworker.0 Request processed in 36us (577us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321138.121079:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 731 00000100:00080000:1.0:1767321138.121102:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321138.121106:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321138.121109:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983247360:0@lo:38:kworker.0 00000100:00100000:0.0:1767321138.129986:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983246848:12345-0@lo:400:kworker.0 Request processed in 12381us (12588us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321138.130003:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133 00000100:00080000:2.0:1767321138.130111:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321138.130117:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321138.130121:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321138.130127:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6381 00000100:00080000:2.0:1767321138.130130:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321138.130137:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321138.130185:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1f80 x1853163983247744/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321138.130322:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983247744:0@lo:38:kworker.0 00000100:00100000:1.0:1767321138.130396:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.130612:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247744 00000100:00100000:0.0:1767321138.130622:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247744 00000100:00100000:0.0:1767321138.130661:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2827 00000100:00100000:0.0:1767321138.130668:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983247744:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321138.130691:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321138.130755:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346100fd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983247744:12345-0@lo:38:kworker.0 Request processed in 86us (357us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321138.130763:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2827 00000100:00080000:2.0:1767321138.130910:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321138.130916:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321138.130920:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983247744:0@lo:38:kworker.0 00010000:00080000:3.0:1767321138.131111:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f2d80 x1853163983246848/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321154 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321138.131123:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983246848:0@lo:400:kworker.0 00010000:00010000:3.0:1767321138.309416:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321138.628677:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983247872:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321138.628751:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.628896:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983247872 00000100:00100000:0.0:1767321138.628906:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983247872 00000100:00100000:0.0:1767321138.628957:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1422 00000100:00100000:0.0:1767321138.628967:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983247872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321138.628997:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321138.629051:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983247872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321138.629058:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1422 00000100:00100000:2.0:1767321138.629329:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983248000:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321138.629371:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321138.629462:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248000 00000100:00100000:0.0:1767321138.629471:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248000 00000100:00100000:0.0:1767321138.629505:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1423 00000100:00100000:0.0:1767321138.629512:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983248000:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321138.629529:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321138.629560:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983248000:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (188us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321138.629567:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1423 00000100:00100000:2.0:1767321138.629685:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983248000:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321138.629867:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983247872:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321139.086892:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:1.0:1767321139.086944:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915720832 offset 0 length 520: 4 00000100:00100000:1.0:1767321139.086970:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321139.087127:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721088 00000100:00100000:2.0:1767321139.087138:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721088 00000100:00080000:1.0:1767321139.087251:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321139.087318:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915720960 00000100:00100000:2.0:1767321139.087323:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915720960 00000100:00100000:0.0:1767321139.091123:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2135 00000100:00100000:0.0:1767321139.091152:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915721088:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321139.091288:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915721088:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 137us (4394us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321139.091296:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2135 00000100:00100000:0.0:1767321139.091324:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2828 00000100:00100000:0.0:1767321139.091329:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915720960:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321139.091361:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321139.091387:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915720960:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (4416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321139.091394:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2828 00010000:00010000:2.0:1767321139.333671:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321140.357193:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321141.384495:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321142.212586:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321117/real 1767321117] req@ffff93463c627100 x1853163983241600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321142 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321142.212628:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93463c627100 x1853163983241600/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321142 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321142.212649:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321142.212670:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321142.212674:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983241600:0@lo:250:kworker.0 00000100:00080000:2.0:1767321142.212739:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321142.212746:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1767321142.212749:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321142.212754:0:54833:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6366 00000100:00080000:2.0:1767321142.212758:0:54833:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321142.212809:0:54833:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321142.212855:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3100 x1853163983248128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321142.212911:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321142.212915:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321142.212917:0:54833:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321142.212921:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6386 00000100:00080000:2.0:1767321142.212923:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321142.212927:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321142.212941:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f1880 x1853163983248384/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321142.212967:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321142.212970:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321142.212972:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321142.212976:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6386 00000100:00080000:2.0:1767321142.212978:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321142.212982:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321142.212992:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163983248640/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321142.212999:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321142.213002:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321142.213004:0:54833:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321142.213007:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6386 00000100:00080000:2.0:1767321142.213010:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321142.213013:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321142.213022:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0000 x1853163983248768/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321142.213065:0:54833:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321142.213069:0:54833:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321142.213070:0:54833:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321142.213074:0:54833:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6386 00000100:00080000:2.0:1767321142.213077:0:54833:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321142.213080:0:54833:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321142.213089:0:54833:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934612b15c00 x1853163983249152/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321142.213114:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983248512:0@lo:400:kworker.0 00000100:00100000:2.0:1767321142.213182:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321142.213233:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321142.213240:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983248896:0@lo:400:kworker.0 00000100:00100000:2.0:1767321142.213265:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321142.213377:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983248256:0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213406:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321142.213437:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321142.213442:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934612b17800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983249024:0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213463:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321142.213492:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248896 00000100:00100000:3.0:1767321142.213500:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248896 00000100:00100000:3.0:1767321142.213546:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249024 00000100:00100000:3.0:1767321142.213550:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249024 00000100:00100000:3.0:1767321142.213570:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1311 00000100:00100000:3.0:1767321142.213579:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983248896:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213627:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983248896:12345-0@lo:400:kworker.0 Request processed in 49us (361us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321142.213635:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1311 00000100:00100000:3.0:1767321142.213652:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1312 00000100:00100000:3.0:1767321142.213658:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983249024:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213687:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983249024:12345-0@lo:400:kworker.0 Request processed in 30us (224us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321142.213692:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1312 00000100:00080000:3.0:1767321142.213707:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:3.0:1767321142.213711:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321142.213720:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248512 00000100:00100000:3.0:1767321142.213723:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248512 00000100:00100000:3.0:1767321142.213753:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248256 00000100:00100000:3.0:1767321142.213756:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248256 00000100:00100000:3.0:1767321142.213774:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136 00000100:00100000:3.0:1767321142.213780:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b16680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983248512:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213809:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b16680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983248512:12345-0@lo:400:kworker.0 Request processed in 30us (627us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321142.213815:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136 00000100:00100000:3.0:1767321142.213826:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2137 00000100:00100000:3.0:1767321142.213831:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983248256:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213857:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983248256:12345-0@lo:400:kworker.0 Request processed in 24us (449us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321142.213865:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2137 00000100:00080000:3.0:1767321142.213884:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321142.213909:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934612b17800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983249024:0@lo:400:kworker.0 00010000:00080000:3.0:1767321142.213920:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f1500 x1853163983248256/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321158 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321142.213929:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983248256:0@lo:400:kworker.0 00000100:00100000:3.0:1767321142.213962:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983248896:0@lo:400:kworker.0 00010000:00080000:3.0:1767321142.213971:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f0380 x1853163983248512/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321158 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321142.213979:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983248512:0@lo:400:kworker.0 00000100:00100000:1.0:1767321142.214183:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983248128:0@lo:250:kworker.0 00000400:00000100:1.0:1767321142.214208:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983248128 offset 0 length 520: 4 00000100:00100000:1.0:1767321142.214229:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983248384:0@lo:38:kworker.0 00000100:00100000:1.0:1767321142.214251:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.214310:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983248640:0@lo:38:kworker.0 00000100:00100000:1.0:1767321142.214340:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.214363:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983248768:0@lo:38:kworker.0 00000100:00100000:1.0:1767321142.214385:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.214402:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934612b15c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983249152:0@lo:38:kworker.0 00000100:00100000:1.0:1767321142.214422:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321142.214447:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248768 00000100:00100000:1.0:1767321142.214451:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248768 00000100:00100000:1.0:1767321142.214472:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 732 00000100:00100000:1.0:1767321142.214476:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983248768:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321142.214494:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321142.214543:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983248768:12345-0@lo:38:kworker.0 Request processed in 65us (156us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321142.214550:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 732 00000100:00080000:1.0:1767321142.214582:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321142.214586:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321142.214589:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983248768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321142.214650:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248384 00000100:00100000:0.0:1767321142.214654:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248384 00000100:00100000:0.0:1767321142.214678:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983248640 00000100:00100000:0.0:1767321142.214680:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983248640 00000100:00100000:0.0:1767321142.214688:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249152 00000100:00100000:0.0:1767321142.214689:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249152 00000100:00100000:0.0:1767321142.214698:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2829 00000100:00100000:0.0:1767321142.214703:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345071d6850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983248384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321142.214713:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321142.214748:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345071d6850 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983248384:12345-0@lo:38:kworker.0 Request processed in 44us (495us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321142.214753:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2829 00000100:00100000:0.0:1767321142.215732:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2830 00000100:00100000:0.0:1767321142.215738:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983248640:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321142.215753:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321142.215792:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96bb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983248640:12345-0@lo:38:kworker.0 Request processed in 53us (1451us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321142.215799:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2830 00000100:00100000:0.0:1767321142.215811:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2831 00000100:00100000:0.0:1767321142.215816:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983249152:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321142.215825:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321142.215846:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96ad80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983249152:12345-0@lo:38:kworker.0 Request processed in 28us (1423us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321142.215855:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2831 00000100:00080000:0.0:1767321142.215925:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321142.215930:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321142.215934:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983248384:0@lo:38:kworker.0 00000100:00080000:0.0:1767321142.215944:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321142.215948:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321142.215951:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983248640:0@lo:38:kworker.0 00000100:00080000:0.0:1767321142.215959:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321142.215963:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321142.215965:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934612b15c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983249152:0@lo:38:kworker.0 00010000:00010000:2.0:1767321142.404545:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321143.428502:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321143.748548:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321143.748567:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983249280:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321143.748645:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321143.748743:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249280 00000100:00100000:0.0:1767321143.748753:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249280 00000100:00100000:0.0:1767321143.748811:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1424 00000100:00100000:0.0:1767321143.748821:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983249280:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321143.748856:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321143.748916:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983249280:12345-0@lo:13:osp-pre-0-1.0 Request processed in 97us (271us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321143.748924:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1424 00000100:00100000:3.0:1767321143.749112:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983249280:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321143.749198:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321143.749207:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983249408:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321143.749234:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321143.749402:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249408 00000100:00100000:2.0:1767321143.749407:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249408 00000100:00100000:2.0:1767321143.749442:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1425 00000100:00100000:2.0:1767321143.749449:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983249408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321143.749467:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321143.749514:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983249408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (279us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321143.749520:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1425 00000100:00100000:3.0:1767321143.749808:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983249408:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321144.203651:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321144.203742:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321144.204071:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321144.214313:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721344 00000100:00100000:3.0:1767321144.214327:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721344 00000100:00100000:3.0:1767321144.214431:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721216 00000100:00100000:3.0:1767321144.214435:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721216 00000100:00100000:3.0:1767321144.214466:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2138 00000100:00100000:3.0:1767321144.214475:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721344:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:0.0:1767321144.214545:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321144.214575:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721344:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 100us (10933us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321144.214582:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2832 00000100:00100000:0.0:1767321144.214590:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915721216:12345-192.168.204.7@tcp:38:kworker.0 00000100:00100000:3.0:1767321144.214595:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2138 00010000:00020000:0.0:1767321144.214614:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321144.214647:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915721216:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 56us (10904us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321144.214658:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2832 00000100:00100000:3.0:1767321144.217034:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721472 00000100:00100000:3.0:1767321144.217049:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721472 00000100:00100000:3.0:1767321144.217115:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2833 00000100:00100000:3.0:1767321144.217125:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721472:12345-192.168.204.7@tcp:17:kworker.0 00000100:00100000:3.0:1767321144.217265:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721472:12345-192.168.204.7@tcp:17:kworker.0 Request processed in 141us (13193us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321144.217293:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2833 00010000:00010000:3.0:1767321144.453299:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321145.476603:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321146.500731:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321147.332942:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321147.332952:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767321147.333035:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983249536:0@lo:400:kworker.0 00000100:00080000:1.0:1767321147.333065:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321147.333071:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321147.333075:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321147.333081:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6390 00000100:00080000:1.0:1767321147.333086:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321147.333093:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321147.333107:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321147.333119:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d968700 x1853163983249664/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321147.333171:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983249664:0@lo:38:kworker.0 00000100:00100000:0.0:1767321147.333215:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321147.333312:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321147.333314:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321147.333316:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321147.333319:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6390 00000100:00080000:1.0:1767321147.333321:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321147.333324:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321147.333333:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d96b100 x1853163983249920/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321147.333456:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249536 00000100:00100000:2.0:1767321147.333463:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249536 00000100:00080000:2.0:1767321147.333485:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321147.333514:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249664 00000100:00100000:2.0:1767321147.333516:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249664 00000100:00100000:2.0:1767321147.333527:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2139 00000100:00100000:2.0:1767321147.333533:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983249536:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.333575:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983249536:12345-0@lo:400:kworker.0 Request processed in 42us (470us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321147.333580:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2139 00000100:00100000:2.0:1767321147.333590:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2834 00000100:00100000:2.0:1767321147.333593:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983249664:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321147.333607:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321147.333808:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983249920:0@lo:38:kworker.0 00000100:00100000:3.0:1767321147.333809:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321147.333820:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983249792:0@lo:400:kworker.0 00000100:00100000:0.0:1767321147.333834:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321147.333856:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321147.333885:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249920 00000100:00080000:1.0:1767321147.333889:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767321147.333890:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249920 00000100:00080000:1.0:1767321147.333894:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321147.333896:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321147.333899:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6390 00000100:00080000:1.0:1767321147.333901:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321147.333905:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321147.333906:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983249792 00000100:00100000:3.0:1767321147.333910:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983249792 00000100:00100000:1.0:1767321147.333921:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d969f80 x1853163983250048/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321147.333945:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2140 00000100:00080000:1.0:1767321147.333954:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1767321147.333956:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983249792:12345-0@lo:400:kworker.0 00000100:00080000:1.0:1767321147.333957:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321147.333960:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321147.333965:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6390 00000100:00080000:1.0:1767321147.333970:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321147.333972:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321147.333980:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463d96b800 x1853163983250432/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321147.334004:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398dc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983249792:12345-0@lo:400:kworker.0 Request processed in 51us (149us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321147.334010:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2140 00000100:00100000:1.0:1767321147.334015:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2835 00000100:00100000:1.0:1767321147.334020:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983249920:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321147.334028:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983250176:0@lo:400:kworker.0 00010000:00020000:1.0:1767321147.334031:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321147.334053:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321147.334077:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983249920:12345-0@lo:38:kworker.0 Request processed in 53us (240us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321147.334084:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2835 00010000:00080000:3.0:1767321147.334090:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463d96aa00 x1853163983249792/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321163 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321147.334104:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983249792:0@lo:400:kworker.0 00000100:00100000:1.0:1767321147.334112:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983250304:0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.334135:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250176 00000100:00100000:1.0:1767321147.334140:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321147.334141:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250176 00000100:00100000:3.0:1767321147.334172:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250304 00000100:00100000:3.0:1767321147.334176:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250304 00000100:00100000:1.0:1767321147.334181:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1313 00000100:00100000:1.0:1767321147.334192:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983250176:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.334212:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1314 00000100:00100000:3.0:1767321147.334225:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983250304:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.334242:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983249664:12345-0@lo:38:kworker.0 Request processed in 645us (1026us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.334249:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2834 00000100:00100000:1.0:1767321147.334256:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983250176:12345-0@lo:400:kworker.0 Request processed in 65us (201us total) trans 0 rc 0/0 00000100:00080000:2.0:1767321147.334264:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321147.334265:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1313 00000100:00080000:2.0:1767321147.334287:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321147.334290:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321147.334294:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983250304:12345-0@lo:400:kworker.0 Request processed in 50us (131us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321147.334302:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1314 00010000:00080000:2.0:1767321147.334337:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463d969500 x1853163983249536/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321163 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321147.334349:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983250304:0@lo:400:kworker.0 00000100:00100000:2.0:1767321147.334350:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983249536:0@lo:400:kworker.0 00000100:00100000:3.0:1767321147.334350:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983250176:0@lo:400:kworker.0 00000100:00080000:0.0:1767321147.335150:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321147.335157:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321147.335161:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983249664:0@lo:38:kworker.0 00000100:00100000:0.0:1767321147.335180:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d969f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983250048:0@lo:38:kworker.0 00000100:00100000:0.0:1767321147.335225:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321147.335254:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d96b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983250432:0@lo:38:kworker.0 00000100:00100000:0.0:1767321147.335291:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321147.335314:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321147.335319:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321147.335323:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983249920:0@lo:38:kworker.0 00000100:00100000:2.0:1767321147.335339:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250432 00000100:00100000:2.0:1767321147.335345:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250432 00000100:00100000:2.0:1767321147.335377:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2836 00000100:00100000:2.0:1767321147.335383:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983250432:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321147.335397:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321147.335438:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983250432:12345-0@lo:38:kworker.0 Request processed in 54us (147us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321147.335443:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2836 00000100:00080000:0.0:1767321147.335506:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321147.335511:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321147.335514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d96b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983250432:0@lo:38:kworker.0 00000100:00100000:1.0:1767321147.335570:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250048 00000100:00100000:1.0:1767321147.335574:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250048 00000100:00100000:1.0:1767321147.335602:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 733 00000100:00100000:1.0:1767321147.335608:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983250048:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321147.335620:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321147.335662:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983250048:12345-0@lo:38:kworker.0 Request processed in 52us (435us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321147.335668:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 733 00000100:00080000:0.0:1767321147.335811:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321147.335816:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321147.335818:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d969f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983250048:0@lo:38:kworker.0 00010000:00010000:3.0:1767321147.525712:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321148.548729:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321148.868700:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321148.868724:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983250688:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321148.868748:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321148.868761:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983250560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321148.868797:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321148.868831:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321148.868905:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250688 00000100:00100000:2.0:1767321148.868922:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250688 00000100:00100000:2.0:1767321148.868967:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250560 00000100:00100000:2.0:1767321148.868970:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250560 00000100:00100000:2.0:1767321148.868984:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1426 00000100:00100000:2.0:1767321148.868991:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983250688:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321148.869019:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321148.869068:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983250688:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321148.869075:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1426 00000100:00100000:2.0:1767321148.869089:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1427 00000100:00100000:2.0:1767321148.869094:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983250560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321148.869108:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321148.869139:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983250560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (309us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321148.869145:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1427 00000100:00100000:3.0:1767321148.869161:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983250688:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321148.869188:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983250560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321149.323807:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321149.324034:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321149.327877:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721728 00000100:00100000:2.0:1767321149.327894:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721728 00000100:00100000:2.0:1767321149.328036:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721600 00000100:00100000:2.0:1767321149.328040:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721600 00000100:00100000:2.0:1767321149.328067:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2141 00000100:00100000:2.0:1767321149.328094:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915721728:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321149.328191:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915721728:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (4397us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321149.328200:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2141 00000100:00100000:2.0:1767321149.328219:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2837 00000100:00100000:2.0:1767321149.328224:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915721600:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321149.328246:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321149.328289:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915721600:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 47us (4239us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321149.328295:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2837 00000100:00080000:1.0:1767321149.333826:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321149.575656:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321150.596513:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321151.620585:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321152.452628:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321152.452639:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321152.452704:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321152.452708:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321152.452712:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321152.452719:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6395 00000100:00080000:0.0:1767321152.452724:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321152.452730:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321152.452753:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55f80 x1853163983250944/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321152.452794:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321152.452798:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321152.452800:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321152.452803:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6395 00000100:00080000:0.0:1767321152.452806:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321152.452810:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321152.452820:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54000 x1853163983251200/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321152.452828:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321152.452831:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321152.452833:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321152.452836:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6395 00000100:00080000:0.0:1767321152.452839:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321152.452842:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321152.452853:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57800 x1853163983251328/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321152.452874:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321152.452877:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321152.452879:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321152.452881:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6395 00000100:00080000:0.0:1767321152.452883:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321152.452886:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321152.452894:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90000 x1853163983251712/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321152.452919:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983251456:0@lo:400:kworker.0 00000100:00100000:0.0:1767321152.452986:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321152.453026:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321152.453033:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983251072:0@lo:400:kworker.0 00000100:00100000:0.0:1767321152.453056:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321152.453178:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983251584:0@lo:400:kworker.0 00000100:00100000:2.0:1767321152.453212:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321152.453249:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321152.453255:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983250816:0@lo:400:kworker.0 00000100:00100000:2.0:1767321152.453299:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321152.453332:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251072 00000100:00100000:2.0:1767321152.453339:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251072 00000100:00080000:2.0:1767321152.453363:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321152.453403:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250816 00000100:00100000:2.0:1767321152.453407:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250816 00000100:00100000:2.0:1767321152.453433:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142 00000100:00100000:2.0:1767321152.453441:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983251072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321152.453505:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983251072:12345-0@lo:400:kworker.0 Request processed in 65us (447us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321152.453512:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142 00000100:00100000:2.0:1767321152.453528:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2143 00000100:00100000:2.0:1767321152.453534:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b17b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983250816:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321152.453565:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b17b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983250816:12345-0@lo:400:kworker.0 Request processed in 31us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321152.453570:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2143 00000100:00080000:2.0:1767321152.453583:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767321152.453608:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e54380 x1853163983250816/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321168 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321152.453622:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983250816:0@lo:400:kworker.0 00010000:00080000:2.0:1767321152.453649:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e56a00 x1853163983251072/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321168 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321152.453658:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983251072:0@lo:400:kworker.0 00000100:00100000:1.0:1767321152.454309:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983250944:0@lo:38:kworker.0 00000100:00100000:1.0:1767321152.454353:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321152.454394:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983251200:0@lo:38:kworker.0 00000100:00100000:1.0:1767321152.454414:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321152.454429:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983251328:0@lo:38:kworker.0 00000100:00100000:1.0:1767321152.454452:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321152.454493:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983251712:0@lo:38:kworker.0 00000100:00100000:1.0:1767321152.454514:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321152.454548:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251328 00000100:00100000:1.0:1767321152.454553:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251328 00000100:00100000:1.0:1767321152.454577:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 734 00000100:00100000:1.0:1767321152.454582:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983251328:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321152.454603:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321152.454639:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983251328:12345-0@lo:38:kworker.0 Request processed in 55us (185us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321152.454645:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 734 00000100:00080000:1.0:1767321152.454673:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321152.454678:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321152.454681:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983251328:0@lo:38:kworker.0 00000100:00100000:2.0:1767321152.454866:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983250944 00000100:00100000:2.0:1767321152.454871:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983250944 00000100:00100000:2.0:1767321152.454909:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251200 00000100:00100000:2.0:1767321152.454913:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251200 00000100:00100000:2.0:1767321152.454923:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251712 00000100:00100000:2.0:1767321152.454925:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251712 00000100:00100000:2.0:1767321152.454933:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2838 00000100:00100000:2.0:1767321152.454937:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983250944:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321152.454983:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321152.455016:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983250944:12345-0@lo:38:kworker.0 Request processed in 77us (664us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321152.455022:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2838 00000100:00100000:2.0:1767321152.455033:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2839 00000100:00100000:2.0:1767321152.455037:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983251200:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321152.455046:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321152.455068:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983251200:12345-0@lo:38:kworker.0 Request processed in 29us (652us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321152.455074:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2839 00000100:00100000:2.0:1767321152.455083:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2840 00000100:00100000:2.0:1767321152.455087:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983251712:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321152.455095:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321152.455130:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983251712:12345-0@lo:38:kworker.0 Request processed in 42us (615us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321152.455149:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2840 00000100:00080000:1.0:1767321152.455507:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321152.455512:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321152.455515:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983250944:0@lo:38:kworker.0 00000100:00080000:1.0:1767321152.455524:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321152.455527:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321152.455530:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983251200:0@lo:38:kworker.0 00000100:00080000:1.0:1767321152.455538:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321152.455541:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321152.455544:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983251712:0@lo:38:kworker.0 00000100:00100000:3.0:1767321152.455641:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251456 00000100:00100000:3.0:1767321152.455647:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251456 00000100:00100000:3.0:1767321152.455701:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251584 00000100:00100000:3.0:1767321152.455704:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251584 00000100:00100000:3.0:1767321152.455726:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1315 00000100:00100000:3.0:1767321152.455733:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983251456:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.455782:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983251456:12345-0@lo:400:kworker.0 Request processed in 50us (2796us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321152.455788:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1315 00000100:00100000:3.0:1767321152.455801:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1316 00000100:00100000:3.0:1767321152.455807:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604772c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983251584:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.455842:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604772c50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983251584:12345-0@lo:400:kworker.0 Request processed in 36us (2630us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321152.455848:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1316 00000100:00100000:3.0:1767321152.456091:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983251584:0@lo:400:kworker.0 00000100:00100000:3.0:1767321152.456133:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983251456:0@lo:400:kworker.0 00000100:00080000:2.0:1767321152.456212:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321152.456217:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321152.646301:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321153.668832:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321153.988572:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983251840:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321153.988650:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321153.988790:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251840 00000100:00100000:0.0:1767321153.988798:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251840 00000100:00100000:0.0:1767321153.988851:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1428 00000100:00100000:0.0:1767321153.988861:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983251840:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321153.988889:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321153.988932:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983251840:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321153.988939:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1428 00000100:00100000:3.0:1767321153.990140:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983251840:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321153.990285:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983251968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321153.990321:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321153.990386:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983251968 00000100:00100000:3.0:1767321153.990391:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983251968 00000100:00100000:3.0:1767321153.990441:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1429 00000100:00100000:3.0:1767321153.990449:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983251968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321153.990469:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321153.992686:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983251968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2226us (2351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321153.992699:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1429 00000100:00100000:2.0:1767321153.992828:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983251968:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321154.443530:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321154.443748:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721984 00000100:00100000:0.0:1767321154.443760:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721984 00000100:00100000:0.0:1767321154.443836:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2144 00000100:00100000:0.0:1767321154.443846:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721984:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321154.443921:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915721984:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (413us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321154.443929:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2144 00000100:00080000:2.0:1767321154.444268:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321154.444479:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321154.444536:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915721856 00000100:00100000:3.0:1767321154.444542:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915721856 00000100:00100000:3.0:1767321154.444570:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2841 00000100:00100000:3.0:1767321154.444575:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b16d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915721856:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321154.444597:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321154.444638:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b16d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915721856:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 62us (159us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321154.444645:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2841 00010000:00010000:2.0:1767321154.693401:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321155.717461:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321156.741650:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321157.575085:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321157.575096:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321157.575154:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321157.575158:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321157.575162:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321157.575169:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6400 00000100:00080000:0.0:1767321157.575174:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321157.575181:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321157.575202:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163983252224/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321157.575244:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321157.575247:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321157.575249:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321157.575253:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6400 00000100:00080000:0.0:1767321157.575255:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321157.575259:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321157.575268:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92d80 x1853163983252480/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321157.575294:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321157.575297:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321157.575299:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321157.575302:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6400 00000100:00080000:0.0:1767321157.575305:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321157.575308:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321157.575318:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163983252608/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321157.575348:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321157.575352:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321157.575354:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321157.575365:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6400 00000100:00080000:0.0:1767321157.575368:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321157.575371:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321157.575380:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d90e00 x1853163983252992/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321157.575472:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983252224:0@lo:38:kworker.0 00000100:00100000:1.0:1767321157.575531:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321157.575587:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983252480:0@lo:38:kworker.0 00000100:00100000:1.0:1767321157.575607:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321157.575623:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983252608:0@lo:38:kworker.0 00000100:00100000:1.0:1767321157.575644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321157.575663:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983252992:0@lo:38:kworker.0 00000100:00100000:1.0:1767321157.575681:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321157.575883:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983252864:0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.575912:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321157.575931:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321157.575936:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983252096:0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.575953:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321157.575973:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983252352:0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.575991:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321157.576014:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321157.576020:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983252736:0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.576039:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321157.576065:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252608 00000100:00100000:2.0:1767321157.576073:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252608 00000100:00100000:2.0:1767321157.576104:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 735 00000100:00100000:2.0:1767321157.576111:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983252608:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321157.576130:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321157.576170:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96aa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983252608:12345-0@lo:38:kworker.0 Request processed in 57us (524us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321157.576177:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 735 00000100:00100000:2.0:1767321157.576198:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252224 00000100:00100000:2.0:1767321157.576201:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252224 00000100:00100000:2.0:1767321157.576212:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252480 00000100:00100000:2.0:1767321157.576213:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252480 00000100:00100000:2.0:1767321157.576220:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252992 00000100:00100000:2.0:1767321157.576222:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252992 00000100:00100000:2.0:1767321157.576229:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252096 00000100:00100000:2.0:1767321157.576231:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252096 00000100:00100000:2.0:1767321157.576255:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252352 00000100:00100000:2.0:1767321157.576258:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252352 00000100:00100000:2.0:1767321157.576285:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145 00000100:00100000:2.0:1767321157.576292:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b16300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983252096:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.576328:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b16300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983252096:12345-0@lo:400:kworker.0 Request processed in 37us (374us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321157.576346:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145 00000100:00100000:1.0:1767321157.576476:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252864 00000100:00100000:1.0:1767321157.576480:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252864 00000100:00100000:1.0:1767321157.576503:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983252736 00000100:00100000:1.0:1767321157.576505:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983252736 00000100:00100000:1.0:1767321157.576518:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767321157.576524:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983252864:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321157.576553:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983252864:12345-0@lo:400:kworker.0 Request processed in 29us (640us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321157.576558:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1317 00000100:00100000:1.0:1767321157.576570:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1318 00000100:00100000:1.0:1767321157.576576:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983252736:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321157.576605:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983252736:12345-0@lo:400:kworker.0 Request processed in 30us (565us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321157.576610:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1318 00000100:00100000:1.0:1767321157.576656:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983252736:0@lo:400:kworker.0 00000100:00080000:1.0:1767321157.576683:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321157.576688:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321157.576691:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983252608:0@lo:38:kworker.0 00000100:00100000:2.0:1767321157.576889:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2146 00000100:00100000:2.0:1767321157.576898:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b17480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983252352:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.576933:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b17480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983252352:12345-0@lo:400:kworker.0 Request processed in 38us (941us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321157.588908:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2842 00000100:00100000:0.0:1767321157.588924:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252224:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321157.588957:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321157.589065:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252224:12345-0@lo:38:kworker.0 Request processed in 144us (13532us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321157.589073:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2842 00000100:00100000:0.0:1767321157.589106:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2843 00000100:00100000:0.0:1767321157.589111:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252480:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321157.589121:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321157.589146:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252480:12345-0@lo:38:kworker.0 Request processed in 34us (13538us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321157.589153:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2843 00000100:00100000:0.0:1767321157.589164:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2844 00000100:00100000:0.0:1767321157.589168:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252992:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321157.589177:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321157.589199:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983252992:12345-0@lo:38:kworker.0 Request processed in 29us (13517us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321157.589204:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2844 00000100:00080000:0.0:1767321157.589253:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321157.589258:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321157.589261:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983252224:0@lo:38:kworker.0 00000100:00080000:0.0:1767321157.589285:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321157.589288:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321157.589291:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983252480:0@lo:38:kworker.0 00000100:00080000:0.0:1767321157.589301:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321157.589304:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321157.589307:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983252992:0@lo:38:kworker.0 00000100:00080000:0.0:1767321157.589379:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321157.589385:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321157.589388:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:0.0:1767321157.589410:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d92300 x1853163983252096/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321173 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321157.589423:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983252096:0@lo:400:kworker.0 00000100:00100000:0.0:1767321157.589435:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983252864:0@lo:400:kworker.0 00010000:00080000:0.0:1767321157.589462:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d93480 x1853163983252352/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321173 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321157.589471:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983252352:0@lo:400:kworker.0 00000100:00100000:2.0:1767321157.590325:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2146 00010000:00010000:2.0:1767321157.764951:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321158.790952:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321159.109445:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983253120:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321159.109528:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321159.109818:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253120 00000100:00100000:1.0:1767321159.109827:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253120 00000100:00100000:1.0:1767321159.109907:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1430 00000100:00100000:0.0:1767321159.109917:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983253248:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321159.109920:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983253120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321159.109955:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321159.109958:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321159.110027:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983253120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 110us (497us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321159.110036:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1430 00000100:00100000:1.0:1767321159.110053:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253248 00000100:00100000:1.0:1767321159.110056:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253248 00000100:00100000:1.0:1767321159.110073:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1431 00000100:00100000:1.0:1767321159.110078:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983253248:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321159.110093:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321159.110132:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983253248:12345-0@lo:13:osp-pre-0-1.0 Request processed in 53us (175us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321159.110138:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1431 00000100:00080000:2.0:1767321159.110294:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321159.110299:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321159.110402:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983253248:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321159.113470:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983253120:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321159.564746:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321159.564802:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321159.564902:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722112 00000100:00100000:2.0:1767321159.564917:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722112 00000100:00100000:3.0:1767321159.564960:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722240 00000100:00100000:3.0:1767321159.564966:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722240 00000100:00100000:2.0:1767321159.564981:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2845 00000100:00100000:2.0:1767321159.564991:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915722112:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321159.565015:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321159.565033:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2147 00000100:00100000:3.0:1767321159.565043:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915722240:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321159.565084:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915722112:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 92us (347us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321159.565092:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2845 00000100:00100000:3.0:1767321159.565101:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915722240:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 57us (296us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321159.565110:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2147 00000100:00080000:0.0:1767321159.565359:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321159.813797:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321160.836464:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321161.860555:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321162.697155:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321162.697168:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321162.697246:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321162.697250:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321162.697254:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321162.697261:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6406 00000100:00080000:0.0:1767321162.697265:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321162.697297:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321162.697300:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321162.697320:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163983253504/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321162.697323:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983253376:0@lo:400:kworker.0 00000100:00080000:0.0:1767321162.697365:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321162.697369:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321162.697371:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321162.697375:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6406 00000100:00080000:0.0:1767321162.697378:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321162.697382:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321162.697396:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93480 x1853163983253760/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321162.697405:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321162.697409:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321162.697410:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321162.697413:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6406 00000100:00080000:0.0:1767321162.697418:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1767321162.697419:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321162.697425:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321162.697505:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253376 00000100:00100000:3.0:1767321162.697513:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253376 00000100:00100000:3.0:1767321162.697577:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148 00000100:00100000:3.0:1767321162.697586:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983253376:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321162.697624:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163983253888/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321162.697640:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ed80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983253376:12345-0@lo:400:kworker.0 Request processed in 54us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321162.697651:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148 00000100:00080000:0.0:1767321162.697653:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321162.697658:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321162.697660:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321162.697664:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6406 00000100:00080000:0.0:1767321162.697667:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321162.697671:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00080000:3.0:1767321162.697674:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321162.697682:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93100 x1853163983254272/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321162.697691:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983254144:0@lo:400:kworker.0 00000100:00100000:3.0:1767321162.697720:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321162.697736:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983254016:0@lo:400:kworker.0 00010000:00080000:3.0:1767321162.697757:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d92d80 x1853163983253376/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321178 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321162.697770:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983253376:0@lo:400:kworker.0 00000100:00100000:3.0:1767321162.697812:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254144 00000100:00100000:3.0:1767321162.697816:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254144 00000100:00100000:3.0:1767321162.697847:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1319 00000100:00100000:3.0:1767321162.697854:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983254144:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321162.697887:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983254144:12345-0@lo:400:kworker.0 Request processed in 34us (166us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321162.697893:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1319 00000100:00080000:3.0:1767321162.697921:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321162.697939:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983254144:0@lo:400:kworker.0 00000100:00100000:0.0:1767321162.698111:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321162.698146:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321162.698151:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983253632:0@lo:400:kworker.0 00000100:00100000:0.0:1767321162.698179:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321162.699025:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983253504:0@lo:38:kworker.0 00000100:00100000:3.0:1767321162.699257:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254016 00000100:00100000:3.0:1767321162.699265:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254016 00000100:00100000:1.0:1767321162.699306:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321162.699342:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983253760:0@lo:38:kworker.0 00000100:00100000:1.0:1767321162.699364:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321162.699381:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983253888:0@lo:38:kworker.0 00000100:00100000:1.0:1767321162.699402:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321162.699423:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253888 00000100:00100000:1.0:1767321162.699427:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253888 00000100:00100000:3.0:1767321162.699428:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1320 00000100:00100000:3.0:1767321162.699439:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983254016:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321162.699474:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321162.699492:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983254016:12345-0@lo:400:kworker.0 Request processed in 55us (1381us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321162.699500:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1320 00000100:00100000:2.0:1767321162.699510:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253632 00000100:00100000:2.0:1767321162.699515:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253632 00000100:00100000:3.0:1767321162.699525:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253504 00000100:00100000:3.0:1767321162.699530:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253504 00000100:00100000:3.0:1767321162.699553:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983253760 00000100:00100000:3.0:1767321162.699556:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983253760 00000100:00100000:1.0:1767321162.699751:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 736 00000100:00100000:1.0:1767321162.699756:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983253888:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321162.699774:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321162.699816:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d969c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983253888:12345-0@lo:38:kworker.0 Request processed in 59us (413us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321162.699827:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 736 00000100:00080000:1.0:1767321162.699864:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321162.699869:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321162.699872:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983253888:0@lo:38:kworker.0 00000100:00100000:1.0:1767321162.699879:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983254272:0@lo:38:kworker.0 00000100:00100000:1.0:1767321162.699902:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321162.702657:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983254016:0@lo:400:kworker.0 00000100:00100000:2.0:1767321162.702734:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254272 00000100:00100000:2.0:1767321162.702742:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254272 00000100:00100000:2.0:1767321162.702790:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2149 00000100:00100000:2.0:1767321162.702802:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983253632:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321162.702876:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983253632:12345-0@lo:400:kworker.0 Request processed in 78us (4695us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321162.702883:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2149 00000100:00100000:2.0:1767321162.702902:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2846 00000100:00100000:2.0:1767321162.702905:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983253504:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321162.702934:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321162.702967:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983253504:12345-0@lo:38:kworker.0 Request processed in 60us (3663us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321162.702973:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2846 00000100:00100000:2.0:1767321162.702987:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2847 00000100:00100000:2.0:1767321162.702991:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d96a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983253760:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321162.702999:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321162.703022:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d96a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983253760:12345-0@lo:38:kworker.0 Request processed in 29us (3657us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321162.703027:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2847 00000100:00100000:2.0:1767321162.703038:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2848 00000100:00100000:2.0:1767321162.703041:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983254272:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321162.703049:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321162.703071:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983254272:12345-0@lo:38:kworker.0 Request processed in 28us (3168us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321162.703076:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2848 00000100:00080000:2.0:1767321162.703098:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767321162.703120:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d90a80 x1853163983253632/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321178 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321162.703130:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983253632:0@lo:400:kworker.0 00000100:00080000:2.0:1767321162.703176:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321162.703181:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321162.703185:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983253504:0@lo:38:kworker.0 00000100:00080000:2.0:1767321162.703194:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321162.703198:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321162.703201:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983253760:0@lo:38:kworker.0 00000100:00080000:2.0:1767321162.703210:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321162.703214:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321162.703218:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983254272:0@lo:38:kworker.0 00010000:00010000:3.0:1767321162.884453:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321163.908856:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321164.228515:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321164.228523:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983254400:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321164.228535:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983254528:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321164.228863:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321164.228986:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254528 00000100:00100000:1.0:1767321164.228994:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254528 00000100:00100000:1.0:1767321164.229060:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1432 00000100:00100000:1.0:1767321164.229069:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983254528:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321164.229098:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321164.229144:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983254528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 75us (280us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321164.229151:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1432 00000100:00080000:3.0:1767321164.229211:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321164.229244:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983254528:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321164.230983:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321164.231250:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254400 00000100:00100000:0.0:1767321164.231258:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254400 00000100:00100000:0.0:1767321164.231337:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1433 00000100:00100000:0.0:1767321164.231346:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983254400:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321164.231377:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767321164.231427:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321164.231436:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983254400:12345-0@lo:13:osp-pre-0-1.0 Request processed in 90us (458us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321164.231445:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1433 00000100:00100000:2.0:1767321164.233389:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983254400:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321164.687361:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:2.0:1767321164.687411:0:7004:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915722368 offset 0 length 520: 4 00000100:00100000:2.0:1767321164.687441:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321164.687599:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722624 00000100:00100000:1.0:1767321164.687611:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722624 00000100:00100000:1.0:1767321164.687690:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722496 00000100:00100000:1.0:1767321164.687693:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722496 00000100:00100000:1.0:1767321164.687712:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2150 00000100:00100000:1.0:1767321164.687722:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b16a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915722624:12345-192.168.204.7@tcp:400:kworker.0 00000100:00080000:3.0:1767321164.687753:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321164.687819:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b16a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915722624:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 89us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321164.687827:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2150 00000100:00100000:1.0:1767321164.687846:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2849 00000100:00100000:1.0:1767321164.687851:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b17100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915722496:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321164.687871:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321164.687910:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b17100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915722496:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 57us (468us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321164.687918:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2849 00010000:00010000:2.0:1767321164.933482:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321165.956710:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321166.981704:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321167.812405:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.812417:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767321167.812540:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983254656:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.812613:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1767321167.812706:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.812711:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321167.812714:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321167.812719:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6411 00000100:00080000:0.0:1767321167.812722:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321167.812729:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321167.812754:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91f80 x1853163983254784/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321167.812799:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.812803:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321167.812805:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321167.812809:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6411 00000100:00080000:0.0:1767321167.812812:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321167.812827:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321167.812839:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93480 x1853163983255040/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321167.812847:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.812850:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321167.812852:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321167.812855:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6411 00000100:00080000:0.0:1767321167.812857:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321167.812860:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321167.812869:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d92a00 x1853163983255168/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321167.812932:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983255424:0@lo:400:kworker.0 00000100:00100000:1.0:1767321167.812965:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321167.813021:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983254912:0@lo:400:kworker.0 00000100:00100000:1.0:1767321167.813044:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321167.813062:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321167.813067:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983255296:0@lo:400:kworker.0 00000100:00100000:1.0:1767321167.813086:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321167.813707:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255424 00000100:00100000:3.0:1767321167.813716:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255424 00000100:00100000:2.0:1767321167.813852:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254656 00000100:00100000:2.0:1767321167.813858:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254656 00000100:00100000:3.0:1767321167.813916:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255296 00000100:00100000:3.0:1767321167.813920:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255296 00000100:00080000:1.0:1767321167.814005:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321167.814010:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321167.814012:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321167.814038:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254912 00000100:00100000:1.0:1767321167.814041:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254912 00000100:00100000:1.0:1767321167.814080:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2151 00000100:00100000:1.0:1767321167.814089:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983254912:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321167.814148:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983254912:12345-0@lo:400:kworker.0 Request processed in 61us (1103us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321167.814154:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2151 00000100:00100000:1.0:1767321167.814184:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321142/real 1767321142] req@ffff9346330f3100 x1853163983248128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321167 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321167.814203:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346330f3100 x1853163983248128/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321167 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321167.814217:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983254784:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.814243:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321167.814287:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983255040:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.814309:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321167.814326:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983255168:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.814348:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321167.814369:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321167.814395:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321167.814398:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983248128:0@lo:250:kworker.0 00000100:00100000:1.0:1767321167.814425:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983254784 00000100:00100000:1.0:1767321167.814429:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983254784 00000100:00100000:1.0:1767321167.814448:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255040 00000100:00100000:1.0:1767321167.814451:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255040 00000100:00100000:1.0:1767321167.814460:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2850 00000100:00100000:1.0:1767321167.814465:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983254784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321167.814483:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321167.814516:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983254784:12345-0@lo:38:kworker.0 Request processed in 50us (273us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321167.814522:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2850 00000100:00100000:1.0:1767321167.814533:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2851 00000100:00100000:1.0:1767321167.814536:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983255040:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321167.814544:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321167.814565:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983255040:12345-0@lo:38:kworker.0 Request processed in 28us (256us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321167.814574:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2851 00010000:00080000:1.0:1767321167.814610:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d93100 x1853163983254912/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321183 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321167.814621:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983254912:0@lo:400:kworker.0 00000100:00080000:1.0:1767321167.814637:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321167.814652:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321167.814656:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321167.814658:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983254784:0@lo:38:kworker.0 00000100:00080000:1.0:1767321167.814665:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321167.814668:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321167.814670:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983255040:0@lo:38:kworker.0 00000100:00100000:1.0:1767321167.814710:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255168 00000100:00100000:1.0:1767321167.814713:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255168 00000100:00100000:1.0:1767321167.814732:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 737 00000100:00100000:1.0:1767321167.814737:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983255168:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321167.814746:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321167.815018:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983255168:12345-0@lo:38:kworker.0 Request processed in 278us (667us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321167.815025:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 737 00000100:00100000:1.0:1767321167.815048:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1321 00000100:00100000:1.0:1767321167.815058:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983255424:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321167.815094:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983255424:12345-0@lo:400:kworker.0 Request processed in 41us (2129us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321167.815099:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1321 00000100:00100000:1.0:1767321167.815131:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983255424:0@lo:400:kworker.0 00000100:00080000:1.0:1767321167.815153:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321167.815156:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321167.815159:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d92a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983255168:0@lo:38:kworker.0 00000100:00080000:0.0:1767321167.815279:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.815285:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321167.815287:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321167.815292:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6411 00000100:00080000:0.0:1767321167.815295:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321167.815299:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321167.815325:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d91500 x1853163983255552/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321167.815350:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321167.815353:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321167.815355:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321167.815358:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6390 00000100:00080000:0.0:1767321167.815360:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321167.815364:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321167.815385:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639d93b80 x1853163983255680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321167.815406:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983255552:0@lo:38:kworker.0 00000100:00100000:0.0:1767321167.815436:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321167.815456:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983255680:0@lo:250:kworker.0 00000400:00000100:0.0:1767321167.815473:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983255680 offset 0 length 520: 4 00000100:00100000:0.0:1767321167.815501:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255552 00000100:00100000:0.0:1767321167.815504:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255552 00000100:00100000:0.0:1767321167.815524:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2852 00000100:00100000:0.0:1767321167.815529:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983255552:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321167.815539:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321167.815568:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983255552:12345-0@lo:38:kworker.0 Request processed in 38us (131us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321167.815574:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2852 00000100:00080000:0.0:1767321167.815598:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321167.815602:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321167.815605:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983255552:0@lo:38:kworker.0 00000100:00100000:2.0:1767321167.815994:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2152 00000100:00100000:2.0:1767321167.816004:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983254656:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321167.816049:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983254656:12345-0@lo:400:kworker.0 Request processed in 48us (3436us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321167.816056:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2152 00010000:00080000:2.0:1767321167.816085:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639d90000 x1853163983254656/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321183 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321167.816096:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983254656:0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.816306:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1322 00000100:00100000:3.0:1767321167.816315:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983255296:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321167.816363:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983255296:12345-0@lo:400:kworker.0 Request processed in 51us (3276us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321167.816369:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1322 00000100:00100000:2.0:1767321167.816453:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d90a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983255296:0@lo:400:kworker.0 00010000:00010000:2.0:1767321168.004679:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321169.035717:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321169.348728:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983255808:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321169.348829:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321169.348992:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321169.349001:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983255936:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321169.349028:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321169.349059:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255808 00000100:00100000:0.0:1767321169.349067:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255808 00000100:00100000:0.0:1767321169.349120:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983255936 00000100:00100000:0.0:1767321169.349124:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983255936 00000100:00100000:0.0:1767321169.349142:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1434 00000100:00100000:0.0:1767321169.349150:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983255808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321169.349180:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321169.349222:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983255808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321169.349229:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1434 00000100:00080000:1.0:1767321169.349233:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321169.349239:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321169.349248:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1435 00000100:00100000:0.0:1767321169.349255:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983255936:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321169.349310:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321169.349359:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983255936:12345-0@lo:13:osp-pre-0-1.0 Request processed in 105us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321169.349365:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1435 00000100:00100000:2.0:1767321169.349517:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983255808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321169.349579:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983255936:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321169.806520:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321169.806614:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321169.806932:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722752 00000100:00100000:3.0:1767321169.806942:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722752 00000100:00100000:3.0:1767321169.806987:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915722880 00000100:00100000:3.0:1767321169.806991:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915722880 00000100:00100000:3.0:1767321169.807044:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2153 00000100:00100000:3.0:1767321169.807056:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915722880:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321169.807072:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2853 00000100:00100000:1.0:1767321169.807079:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915722752:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321169.807096:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321169.807150:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321169.807210:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915722880:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 155us (594us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321169.807219:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2153 00000100:00100000:1.0:1767321169.807236:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915722752:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 153us (724us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321169.807243:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2853 00010000:00010000:2.0:1767321170.052518:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321171.077307:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321172.101315:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321172.933062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321172.933074:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321172.933147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321172.933152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321172.933156:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321172.933165:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6416 00000100:00080000:1.0:1767321172.933170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321172.933177:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321172.933201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c625500 x1853163983256192/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321172.933235:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1767321172.933236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321172.933242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321172.933244:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321172.933248:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6416 00000100:00080000:1.0:1767321172.933251:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321172.933257:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321172.933258:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983256064:0@lo:400:kworker.0 00000100:00100000:1.0:1767321172.933268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624e00 x1853163983256448/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321172.933309:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321172.933312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321172.933314:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321172.933317:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6416 00000100:00080000:1.0:1767321172.933319:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321172.933323:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321172.933336:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c627b80 x1853163983256576/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321172.933360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321172.933363:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321172.933365:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321172.933368:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6416 00000100:00080000:1.0:1767321172.933371:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321172.933374:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321172.933380:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321172.933386:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463c624380 x1853163983256960/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321172.933422:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983256832:0@lo:400:kworker.0 00000100:00100000:1.0:1767321172.933426:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983256320:0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.933461:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321172.933471:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.933501:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256064 00000100:00100000:1.0:1767321172.933502:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1767321172.933509:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256064 00000100:00100000:1.0:1767321172.933510:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983256704:0@lo:400:kworker.0 00000100:00100000:1.0:1767321172.933538:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321172.933567:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256320 00000100:00100000:3.0:1767321172.933571:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256320 00000100:00100000:1.0:1767321172.933587:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256832 00000100:00100000:1.0:1767321172.933592:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256832 00000100:00100000:3.0:1767321172.933607:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154 00000100:00100000:3.0:1767321172.933619:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983256064:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321172.933630:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256704 00000100:00100000:1.0:1767321172.933635:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256704 00000100:00100000:1.0:1767321172.933660:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1323 00000100:00100000:1.0:1767321172.933668:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983256832:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.933685:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983256064:12345-0@lo:400:kworker.0 Request processed in 68us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321172.933693:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154 00000100:00100000:1.0:1767321172.933710:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983256832:12345-0@lo:400:kworker.0 Request processed in 43us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321172.933715:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2155 00000100:00100000:1.0:1767321172.933736:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1323 00000100:00100000:3.0:1767321172.933736:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983256320:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321172.933758:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1324 00000100:00100000:1.0:1767321172.933768:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983256704:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.933781:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983256320:12345-0@lo:400:kworker.0 Request processed in 57us (309us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321172.933788:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2155 00000100:00100000:1.0:1767321172.933806:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983256704:12345-0@lo:400:kworker.0 Request processed in 40us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321172.933814:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1324 00000100:00100000:3.0:1767321172.933844:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983256704:0@lo:400:kworker.0 00010000:00080000:3.0:1767321172.933858:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c624700 x1853163983256320/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321188 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321172.933867:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983256320:0@lo:400:kworker.0 00000100:00100000:3.0:1767321172.933894:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983256832:0@lo:400:kworker.0 00010000:00080000:3.0:1767321172.933906:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463c626300 x1853163983256064/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321188 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321172.933916:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c626300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983256064:0@lo:400:kworker.0 00000100:00100000:0.0:1767321172.933934:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983256192:0@lo:38:kworker.0 00000100:00080000:2.0:1767321172.934008:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321172.934013:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321172.934016:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321172.934022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321172.934048:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983256448:0@lo:38:kworker.0 00000100:00100000:0.0:1767321172.934071:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321172.934097:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983256576:0@lo:38:kworker.0 00000100:00100000:0.0:1767321172.934120:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321172.934147:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983256960:0@lo:38:kworker.0 00000100:00100000:0.0:1767321172.934166:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321172.934320:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256576 00000100:00100000:2.0:1767321172.934326:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256576 00000100:00100000:2.0:1767321172.934357:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 738 00000100:00100000:2.0:1767321172.934362:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983256576:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321172.934380:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321172.934422:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983256576:12345-0@lo:38:kworker.0 Request processed in 59us (301us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321172.934428:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 738 00000100:00080000:2.0:1767321172.934456:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321172.934460:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321172.934462:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c627b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983256576:0@lo:38:kworker.0 00000100:00100000:3.0:1767321172.934530:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256192 00000100:00100000:3.0:1767321172.934535:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256192 00000100:00100000:3.0:1767321172.934562:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256448 00000100:00100000:3.0:1767321172.934565:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256448 00000100:00100000:3.0:1767321172.934579:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983256960 00000100:00100000:3.0:1767321172.934582:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983256960 00000100:00100000:3.0:1767321172.934590:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2854 00000100:00100000:3.0:1767321172.934594:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.934605:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321172.934637:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256192:12345-0@lo:38:kworker.0 Request processed in 41us (616us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.934643:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2854 00000100:00100000:3.0:1767321172.934656:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2855 00000100:00100000:3.0:1767321172.934660:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256448:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.934669:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321172.934691:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256448:12345-0@lo:38:kworker.0 Request processed in 30us (620us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.934696:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2855 00000100:00100000:3.0:1767321172.934706:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2856 00000100:00100000:3.0:1767321172.934710:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256960:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321172.934718:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321172.934753:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983256960:12345-0@lo:38:kworker.0 Request processed in 41us (585us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321172.934759:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2856 00000100:00080000:3.0:1767321172.934787:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.934791:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321172.934794:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c625500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983256192:0@lo:38:kworker.0 00000100:00080000:3.0:1767321172.934803:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.934806:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321172.934809:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983256448:0@lo:38:kworker.0 00000100:00080000:3.0:1767321172.934816:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321172.934820:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321172.934822:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463c624380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983256960:0@lo:38:kworker.0 00010000:00010000:2.0:1767321173.124694:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321174.148589:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321174.468646:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321174.468672:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983257088:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321174.468798:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321174.469012:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983257216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321174.469045:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321174.469246:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257088 00000100:00100000:1.0:1767321174.469255:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257088 00000100:00100000:1.0:1767321174.469344:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257216 00000100:00100000:1.0:1767321174.469348:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257216 00000100:00100000:1.0:1767321174.469367:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1436 00000100:00100000:1.0:1767321174.469376:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983257088:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321174.469404:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321174.469450:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983257088:12345-0@lo:13:osp-pre-0-1.0 Request processed in 75us (652us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321174.469462:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1436 00000100:00100000:1.0:1767321174.469480:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1437 00000100:00100000:1.0:1767321174.469487:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983257216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321174.469503:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321174.469536:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983257216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (491us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321174.469543:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1437 00000100:00100000:3.0:1767321174.469769:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d91f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983257088:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321174.469871:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983257216:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1767321174.469931:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321174.469936:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321174.924860:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321174.925858:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723008 00000100:00100000:3.0:1767321174.925871:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723008 00000100:00100000:3.0:1767321174.925930:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2857 00000100:00100000:3.0:1767321174.925938:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915723008:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321174.925960:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321174.926024:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915723008:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 85us (1173us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321174.926041:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2857 00000100:00100000:0.0:1767321174.926573:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321174.928374:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723136 00000100:00100000:2.0:1767321174.928388:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723136 00000100:00100000:2.0:1767321174.928490:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2156 00000100:00100000:2.0:1767321174.928501:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915723136:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321174.928586:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915723136:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 87us (2014us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321174.928595:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2156 00000100:00080000:0.0:1767321174.931592:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321175.172770:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321176.196460:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321177.221493:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321178.053096:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321178.053108:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1767321178.053163:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321178.053166:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321178.053170:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321178.053177:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6421 00000100:00080000:3.0:1767321178.053181:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321178.053186:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321178.053204:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93463398d180 x1853163983257472/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321178.053232:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321178.053235:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321178.053236:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321178.053239:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6421 00000100:00080000:3.0:1767321178.053240:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321178.053243:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321178.053251:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3800 x1853163983257728/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321178.053256:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321178.053258:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321178.053259:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321178.053262:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6421 00000100:00080000:3.0:1767321178.053263:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321178.053266:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321178.053295:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f3480 x1853163983257856/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321178.053314:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321178.053317:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321178.053319:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767321178.053323:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983257344:0@lo:400:kworker.0 00000100:00080000:3.0:1767321178.053324:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6421 00000100:00080000:3.0:1767321178.053330:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321178.053335:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321178.053347:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346330f0700 x1853163983258240/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321178.053375:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983257600:0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053407:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321178.053441:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321178.053460:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321178.053469:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321178.053470:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983258112:0@lo:400:kworker.0 00000100:00100000:3.0:1767321178.053481:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983257984:0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053509:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321178.053520:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257344 00000100:00100000:3.0:1767321178.053528:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257344 00000100:00100000:0.0:1767321178.053556:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258112 00000100:00100000:0.0:1767321178.053563:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258112 00000100:00100000:3.0:1767321178.053576:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257600 00000100:00100000:3.0:1767321178.053580:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257600 00000100:00100000:3.0:1767321178.053616:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157 00000100:00100000:0.0:1767321178.053619:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1325 00000100:00100000:3.0:1767321178.053642:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983257344:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053662:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983258112:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321178.053694:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983257344:12345-0@lo:400:kworker.0 Request processed in 68us (288us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321178.053701:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157 00000100:00100000:0.0:1767321178.053706:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d91500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983258112:12345-0@lo:400:kworker.0 Request processed in 77us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321178.053714:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1325 00000100:00100000:3.0:1767321178.053720:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2158 00000100:00100000:3.0:1767321178.053727:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983257600:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321178.053736:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321178.053742:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321178.053750:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983257600:12345-0@lo:400:kworker.0 Request processed in 24us (311us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321178.053755:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2158 00000100:00100000:0.0:1767321178.053777:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983258112:0@lo:400:kworker.0 00010000:00080000:0.0:1767321178.053789:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93463398f480 x1853163983257344/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321194 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321178.053798:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983257344:0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053835:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:0.0:1767321178.053859:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346330f3100 x1853163983257600/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321194 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321178.053867:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983257600:0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053882:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257984 00000100:00100000:0.0:1767321178.053885:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257984 00000100:00100000:0.0:1767321178.053908:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1326 00000100:00100000:0.0:1767321178.053913:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983257984:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321178.053943:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983257984:12345-0@lo:400:kworker.0 Request processed in 29us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321178.053950:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1326 00000100:00080000:0.0:1767321178.053963:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321178.053989:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983257984:0@lo:400:kworker.0 00000100:00100000:2.0:1767321178.054243:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983257472:0@lo:38:kworker.0 00000100:00100000:2.0:1767321178.054300:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321178.054326:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983257728:0@lo:38:kworker.0 00000100:00100000:2.0:1767321178.054347:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321178.054367:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257472 00000100:00100000:2.0:1767321178.054372:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983257856:0@lo:38:kworker.0 00000100:00100000:1.0:1767321178.054374:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257472 00000100:00100000:1.0:1767321178.054399:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257728 00000100:00100000:1.0:1767321178.054403:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257728 00000100:00100000:2.0:1767321178.054405:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321178.054424:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2858 00000100:00100000:1.0:1767321178.054430:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b16680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983257472:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321178.054449:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983258240:0@lo:38:kworker.0 00010000:00020000:1.0:1767321178.054451:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321178.054489:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321178.054501:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b16680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983257472:12345-0@lo:38:kworker.0 Request processed in 69us (198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321178.054509:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2858 00000100:00080000:2.0:1767321178.054519:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767321178.054523:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258240 00000100:00080000:2.0:1767321178.054526:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321178.054529:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258240 00000100:00100000:2.0:1767321178.054533:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463398d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983257472:0@lo:38:kworker.0 00000100:00100000:1.0:1767321178.054547:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2859 00000100:00100000:1.0:1767321178.054563:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b17800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983257728:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321178.054571:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2860 00010000:00020000:1.0:1767321178.054575:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321178.054605:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b17800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983257728:12345-0@lo:38:kworker.0 Request processed in 51us (257us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321178.054712:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983257856 00000100:00100000:0.0:1767321178.054717:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983257856 00000100:00100000:0.0:1767321178.054739:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 739 00000100:00100000:0.0:1767321178.054744:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983257856:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321178.054754:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321178.054780:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983257856:12345-0@lo:38:kworker.0 Request processed in 36us (379us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321178.054785:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 739 00000100:00100000:2.0:1767321178.054912:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983258240:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321178.054922:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321178.054947:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983258240:12345-0@lo:38:kworker.0 Request processed in 35us (459us total) trans 0 rc -19/-19 00000100:00080000:0.0:1767321178.055002:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321178.055006:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321178.055009:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983257728:0@lo:38:kworker.0 00000100:00080000:0.0:1767321178.055017:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321178.055020:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321178.055023:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983257856:0@lo:38:kworker.0 00000100:00080000:0.0:1767321178.055030:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321178.055033:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321178.055035:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983258240:0@lo:38:kworker.0 00000100:00100000:1.0:1767321178.059689:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2859 00000100:00100000:2.0:1767321178.060060:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2860 00010000:00010000:3.0:1767321178.244760:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321179.268501:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321179.591380:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321179.591404:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983258368:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321179.591481:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321179.591497:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1767321179.591511:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983258496:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321179.591529:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258368 00000100:00100000:3.0:1767321179.591538:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258368 00000100:00100000:0.0:1767321179.591566:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321179.591596:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258496 00000100:00100000:3.0:1767321179.591610:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258496 00000100:00100000:3.0:1767321179.591631:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1438 00000100:00100000:3.0:1767321179.591642:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983258368:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321179.591673:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321179.591719:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983258368:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321179.591727:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1438 00000100:00100000:3.0:1767321179.591745:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1439 00000100:00100000:3.0:1767321179.591752:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983258496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321179.591769:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321179.591807:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983258496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321179.591813:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1439 00000100:00100000:0.0:1767321179.595110:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983258368:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321179.595542:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346330f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983258496:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321180.045063:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321180.045112:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321180.045139:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723264 00000100:00100000:2.0:1767321180.045149:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723264 00000100:00100000:2.0:1767321180.045192:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723392 00000100:00100000:2.0:1767321180.045196:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723392 00000100:00100000:2.0:1767321180.045252:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2159 00000100:00100000:2.0:1767321180.045262:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915723392:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321180.045379:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915723392:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 118us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321180.045387:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2159 00000100:00100000:2.0:1767321180.045406:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2861 00000100:00100000:2.0:1767321180.045410:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915723264:12345-192.168.204.7@tcp:38:kworker.0 00000100:00080000:0.0:1767321180.045428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00020000:2.0:1767321180.045430:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321180.045454:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fbb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915723264:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (401us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321180.045461:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2861 00010000:00010000:2.0:1767321180.292790:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321181.316984:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321182.340930:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321183.172617:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321183.172628:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321183.172690:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321183.172694:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321183.172697:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321183.172707:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6426 00000100:00080000:2.0:1767321183.172712:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321183.172719:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321183.172731:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983258624:0@lo:400:kworker.0 00000100:00100000:2.0:1767321183.172742:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e57800 x1853163983258752/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321183.172791:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321183.172795:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321183.172797:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321183.172802:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6426 00000100:00100000:0.0:1767321183.172803:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321183.172808:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321183.172813:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321183.172838:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e54380 x1853163983259008/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321183.172848:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321183.172851:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321183.172853:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321183.172857:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6426 00000100:00080000:2.0:1767321183.172859:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321183.172864:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321183.172877:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163983259136/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321183.172912:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321183.172915:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321183.172917:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321183.172922:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6426 00000100:00080000:2.0:1767321183.172925:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321183.172929:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321183.172940:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983258752:0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.172942:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dca80 x1853163983259520/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321183.172973:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983259264:0@lo:400:kworker.0 00000100:00100000:1.0:1767321183.172979:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.173013:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983259008:0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173019:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.173048:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321183.173053:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321183.173061:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983258880:0@lo:400:kworker.0 00000100:00100000:1.0:1767321183.173073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983259136:0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173091:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.173106:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.173142:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983259520:0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173143:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259264 00000100:00100000:2.0:1767321183.173153:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259264 00000100:00100000:1.0:1767321183.173171:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321183.173210:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259136 00000100:00100000:1.0:1767321183.173214:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259136 00000100:00100000:2.0:1767321183.173221:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1327 00000100:00100000:2.0:1767321183.173231:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983259264:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321183.173244:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 740 00000100:00100000:1.0:1767321183.173251:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983259136:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173303:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983259264:12345-0@lo:400:kworker.0 Request processed in 73us (287us total) trans 0 rc 0/0 00010000:00020000:1.0:1767321183.173310:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321183.173314:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1327 00000100:00100000:2.0:1767321183.173342:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258624 00000100:00100000:2.0:1767321183.173347:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258624 00000100:00100000:1.0:1767321183.173357:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983259136:12345-0@lo:38:kworker.0 Request processed in 104us (250us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321183.173365:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 740 00000100:00100000:2.0:1767321183.173369:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258752 00000100:00100000:2.0:1767321183.173373:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258752 00000100:00100000:2.0:1767321183.173389:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259008 00000100:00100000:1.0:1767321183.173391:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983258880 00000100:00100000:2.0:1767321183.173393:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259008 00000100:00100000:1.0:1767321183.173396:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983258880 00000100:00100000:2.0:1767321183.173405:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259520 00000100:00100000:2.0:1767321183.173410:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259520 00000100:00100000:1.0:1767321183.173424:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160 00000100:00100000:2.0:1767321183.173431:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2161 00000100:00100000:1.0:1767321183.173436:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983258624:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321183.173445:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983258880:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321183.173475:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983258624:12345-0@lo:400:kworker.0 Request processed in 42us (673us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321183.173487:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dfb80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983258880:12345-0@lo:400:kworker.0 Request processed in 46us (396us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321183.173498:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160 00000100:00100000:2.0:1767321183.173580:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2161 00010000:00080000:0.0:1767321183.173591:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e55f80 x1853163983258624/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321199 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321183.173599:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2862 00000100:00100000:2.0:1767321183.173607:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983258752:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321183.173611:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983258624:0@lo:400:kworker.0 00010000:00020000:2.0:1767321183.173621:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321183.173629:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1767321183.173639:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983259392:0@lo:400:kworker.0 00000100:00100000:2.0:1767321183.173650:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983258752:12345-0@lo:38:kworker.0 Request processed in 42us (671us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321183.173658:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2862 00000100:00100000:0.0:1767321183.173668:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321183.173673:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2863 00000100:00100000:2.0:1767321183.173679:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983259008:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321183.173689:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321183.173719:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983259008:12345-0@lo:38:kworker.0 Request processed in 39us (671us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321183.173723:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2864 00000100:00100000:0.0:1767321183.173733:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983259520:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173733:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2863 00010000:00020000:0.0:1767321183.173748:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1767321183.173758:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:2.0:1767321183.173764:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321183.173777:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259392 00000100:00100000:0.0:1767321183.173781:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c627b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983259520:12345-0@lo:38:kworker.0 Request processed in 47us (610us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321183.173782:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259392 00000100:00100000:0.0:1767321183.173792:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2864 00000100:00100000:2.0:1767321183.173815:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1328 00000100:00100000:2.0:1767321183.173825:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983259392:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321183.173832:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321183.173839:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321183.173843:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983259520:0@lo:38:kworker.0 00000100:00080000:0.0:1767321183.173855:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321183.173859:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321183.173863:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e57800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983258752:0@lo:38:kworker.0 00000100:00100000:2.0:1767321183.173868:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983259392:12345-0@lo:400:kworker.0 Request processed in 44us (199us total) trans 0 rc 0/0 00000100:00080000:0.0:1767321183.173876:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1767321183.173877:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1328 00000100:00080000:0.0:1767321183.173884:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321183.173888:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983259008:0@lo:38:kworker.0 00000100:00080000:2.0:1767321183.173897:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321183.173900:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321183.173906:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321183.173909:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983259136:0@lo:38:kworker.0 00010000:00080000:2.0:1767321183.173920:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934639e56a00 x1853163983258880/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321199 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321183.173931:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983258880:0@lo:400:kworker.0 00000100:00100000:0.0:1767321183.173947:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983259392:0@lo:400:kworker.0 00000100:00100000:2.0:1767321183.173951:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983259264:0@lo:400:kworker.0 00010000:00010000:2.0:1767321183.368458:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321184.389968:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321184.708547:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321184.708576:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983259648:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321184.708655:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321184.710020:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259648 00000100:00100000:3.0:1767321184.710037:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259648 00000100:00100000:3.0:1767321184.710088:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1440 00000100:00100000:3.0:1767321184.710098:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983259648:12345-0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321184.710366:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00002000:00000024:3.0:1767321184.710374:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321184.710378:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983259776:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321184.710426:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321184.710528:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983259648:12345-0@lo:13:osp-pre-0-1.0 Request processed in 431us (1872us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321184.710536:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1440 00000100:00100000:3.0:1767321184.710554:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259776 00000100:00100000:3.0:1767321184.710557:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259776 00000100:00100000:3.0:1767321184.710592:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1441 00000100:00100000:3.0:1767321184.710599:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983259776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321184.710617:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321184.710645:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983259776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (219us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321184.710651:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1441 00000100:00100000:0.0:1767321184.710748:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983259648:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321184.710748:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983259776:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321185.166405:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321185.166567:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321185.168041:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723520 00000100:00100000:3.0:1767321185.168054:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723520 00000100:00100000:3.0:1767321185.168098:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723648 00000100:00100000:3.0:1767321185.168102:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723648 00000100:00100000:3.0:1767321185.168158:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2162 00000100:00100000:3.0:1767321185.168165:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915723648:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321185.168238:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915723648:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 72us (1669us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321185.168246:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2162 00000100:00100000:3.0:1767321185.168265:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2865 00000100:00100000:3.0:1767321185.168285:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915723520:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321185.168302:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321185.168322:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915723520:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 52us (1930us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321185.168326:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2865 00000100:00080000:2.0:1767321185.168414:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321185.414224:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321186.437560:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321187.460542:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321188.292728:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321188.292738:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321188.292801:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321188.292806:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321188.292809:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321188.292816:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6431 00000100:00080000:2.0:1767321188.292820:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321188.292826:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321188.292847:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dce00 x1853163983260032/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321188.292873:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321188.292876:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321188.292878:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321188.292882:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6431 00000100:00080000:2.0:1767321188.292884:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321188.292888:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321188.292897:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddc00 x1853163983260288/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321188.292906:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321188.292909:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321188.292911:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321188.292914:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6431 00000100:00080000:2.0:1767321188.292916:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321188.292919:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321188.292929:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382dd180 x1853163983260416/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321188.293127:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983259904:0@lo:400:kworker.0 00000100:00100000:3.0:1767321188.293191:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321188.293292:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983260672:0@lo:400:kworker.0 00000100:00100000:1.0:1767321188.293322:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983260160:0@lo:400:kworker.0 00000100:00100000:0.0:1767321188.293329:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321188.293357:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321188.293374:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983260544:0@lo:400:kworker.0 00000100:00100000:1.0:1767321188.293394:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983259904 00000100:00100000:1.0:1767321188.293401:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983259904 00000100:00100000:0.0:1767321188.293405:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321188.293462:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260160 00000100:00100000:1.0:1767321188.293466:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260160 00000100:00080000:2.0:1767321188.293491:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321188.293497:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321188.293499:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321188.293502:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6431 00000100:00080000:2.0:1767321188.293505:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321188.293509:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321188.293524:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346382ddf80 x1853163983260800/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321188.293548:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260672 00000100:00100000:2.0:1767321188.293551:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260672 00000100:00100000:2.0:1767321188.293580:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260544 00000100:00100000:2.0:1767321188.293583:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260544 00000100:00100000:2.0:1767321188.293608:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1329 00000100:00100000:2.0:1767321188.293618:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983260672:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321188.293671:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983260672:12345-0@lo:400:kworker.0 Request processed in 54us (343us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321188.293678:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1329 00000100:00100000:2.0:1767321188.293694:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1330 00000100:00100000:2.0:1767321188.293700:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983260544:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321188.293730:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983260544:12345-0@lo:400:kworker.0 Request processed in 31us (328us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321188.293736:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1330 00000100:00080000:2.0:1767321188.293751:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321188.293755:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321188.293757:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321188.295168:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983260672:0@lo:400:kworker.0 00000100:00100000:3.0:1767321188.295218:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983260544:0@lo:400:kworker.0 00000100:00100000:3.0:1767321188.295262:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983260032:0@lo:38:kworker.0 00000100:00100000:3.0:1767321188.295334:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321188.305885:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983260288:0@lo:38:kworker.0 00000100:00100000:3.0:1767321188.306022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321188.306114:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260032 00000100:00100000:0.0:1767321188.306123:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260032 00000100:00100000:0.0:1767321188.306164:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2163 00000100:00100000:0.0:1767321188.306178:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983259904:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321188.306200:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2164 00000100:00100000:0.0:1767321188.306248:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983259904:12345-0@lo:400:kworker.0 Request processed in 75us (13056us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321188.306255:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2163 00000100:00100000:0.0:1767321188.306289:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2866 00000100:00100000:0.0:1767321188.306293:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983260032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321188.306312:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321188.306343:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983260032:12345-0@lo:38:kworker.0 Request processed in 48us (11010us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321188.306349:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2866 00010000:00080000:0.0:1767321188.306404:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382df100 x1853163983259904/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321204 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321188.306419:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983259904:0@lo:400:kworker.0 00000100:00100000:1.0:1767321188.306558:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983260160:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321188.306612:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983260160:12345-0@lo:400:kworker.0 Request processed in 402us (13253us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321188.306621:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2164 00010000:00080000:0.0:1767321188.306642:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346382de680 x1853163983260160/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321204 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321188.306654:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983260160:0@lo:400:kworker.0 00000100:00100000:3.0:1767321188.319484:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983260416:0@lo:38:kworker.0 00000100:00100000:3.0:1767321188.319541:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321188.319608:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260416 00000100:00100000:0.0:1767321188.319616:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260416 00000100:00100000:0.0:1767321188.319656:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 741 00000100:00100000:0.0:1767321188.319663:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983260416:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321188.319684:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321188.319720:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983260416:12345-0@lo:38:kworker.0 Request processed in 57us (179us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321188.319727:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 741 00000100:00100000:2.0:1767321188.319776:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260288 00000100:00100000:2.0:1767321188.319781:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260288 00000100:00100000:2.0:1767321188.319806:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2867 00000100:00100000:2.0:1767321188.319821:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983260288:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321188.319831:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321188.319857:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983260288:12345-0@lo:38:kworker.0 Request processed in 37us (13892us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321188.319863:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2867 00000100:00080000:3.0:1767321188.322453:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321188.322467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321188.322471:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983260416:0@lo:38:kworker.0 00000100:00100000:3.0:1767321188.322483:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983260800:0@lo:38:kworker.0 00000100:00100000:3.0:1767321188.322537:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321188.322607:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321188.322611:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321188.322614:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983260032:0@lo:38:kworker.0 00000100:00080000:3.0:1767321188.322623:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321188.322626:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321188.322629:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983260288:0@lo:38:kworker.0 00000100:00100000:2.0:1767321188.322724:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260800 00000100:00100000:2.0:1767321188.322730:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260800 00000100:00100000:2.0:1767321188.322774:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2868 00000100:00100000:2.0:1767321188.322782:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983260800:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321188.322802:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321188.322859:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983260800:12345-0@lo:38:kworker.0 Request processed in 77us (321us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321188.322866:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2868 00000100:00080000:3.0:1767321188.322930:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321188.322943:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321188.322946:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983260800:0@lo:38:kworker.0 00010000:00010000:2.0:1767321188.484613:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321189.263687:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000400:00000100:0.0:1767321189.263738:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915723776 offset 0 length 520: 4 00000100:00100000:0.0:1767321189.263767:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321189.263919:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724032 00000100:00100000:2.0:1767321189.263929:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724032 00000100:00100000:2.0:1767321189.264002:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915723904 00000100:00100000:2.0:1767321189.264006:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915723904 00000100:00080000:3.0:1767321189.264025:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321189.264027:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2165 00000100:00100000:2.0:1767321189.264039:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915724032:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321189.264122:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fb100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915724032:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (447us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321189.264130:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2165 00000100:00100000:2.0:1767321189.264147:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2869 00000100:00100000:2.0:1767321189.264152:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915723904:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321189.264171:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321189.264195:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915723904:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (428us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321189.264201:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2869 00010000:00010000:2.0:1767321189.508794:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321189.830077:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321189.830112:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983260928:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321189.830186:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321189.830469:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983260928 00000100:00100000:0.0:1767321189.830479:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983260928 00000100:00100000:0.0:1767321189.830608:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1442 00000100:00100000:0.0:1767321189.830619:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983260928:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321189.830650:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321189.830703:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983260928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 86us (516us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321189.830710:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1442 00000100:00100000:1.0:1767321189.831006:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983260928:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321189.838969:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321189.846843:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983261056:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321189.846926:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321189.847226:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261056 00000100:00100000:0.0:1767321189.847236:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261056 00000100:00100000:0.0:1767321189.847315:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1443 00000100:00100000:0.0:1767321189.847325:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983261056:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321189.847361:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321189.847439:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983261056:12345-0@lo:13:osp-pre-0-1.0 Request processed in 106us (503us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321189.847447:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1443 00000100:00100000:1.0:1767321189.847682:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983261056:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767321190.532898:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321191.556613:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321192.452864:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321167/real 1767321167] req@ffff934639d93b80 x1853163983255680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321192 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321192.452899:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934639d93b80 x1853163983255680/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321192 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321192.452919:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321192.452948:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:3.0:1767321192.452952:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639d93b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983255680:0@lo:250:kworker.0 00000100:00080000:0.0:1767321192.453033:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.453038:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321192.453042:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.453047:0:65947:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6416 00000100:00080000:0.0:1767321192.453050:0:65947:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.453057:0:65947:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321192.453093:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f9180 x1853163983261184/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.453158:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.453162:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.453164:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.453168:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6436 00000100:00080000:0.0:1767321192.453170:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.453183:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321192.453199:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163983261440/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321192.453219:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767321192.453226:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.453231:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.453233:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1767321192.453236:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983261312:0@lo:400:kworker.0 00000100:00080000:0.0:1767321192.453240:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6436 00000100:00080000:0.0:1767321192.453245:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.453250:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321192.453264:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8700 x1853163983261696/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.453331:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.453335:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.453336:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.453340:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6436 00000100:00080000:0.0:1767321192.453342:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.453346:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321192.453370:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163983261824/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321192.453396:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321192.453400:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321192.453402:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321192.453406:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6436 00000100:00080000:0.0:1767321192.453408:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321192.453411:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321192.453422:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934639e55180 x1853163983262208/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321192.453490:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.453601:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983262080:0@lo:400:kworker.0 00000100:00100000:2.0:1767321192.453627:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321192.453685:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261312 00000100:00100000:0.0:1767321192.453693:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261312 00000100:00100000:1.0:1767321192.453814:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983261568:0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.453849:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321192.453883:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321192.453891:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983261952:0@lo:400:kworker.0 00000100:00100000:2.0:1767321192.453900:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261568 00000100:00100000:2.0:1767321192.453906:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261568 00000100:00100000:1.0:1767321192.453918:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321192.453977:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166 00000100:00100000:0.0:1767321192.453986:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983261312:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321192.454048:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983261312:12345-0@lo:400:kworker.0 Request processed in 63us (558us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321192.454059:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166 00000100:00100000:3.0:1767321192.454076:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262080 00000100:00100000:3.0:1767321192.454082:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262080 00010000:00080000:1.0:1767321192.454149:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8000 x1853163983261312/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321208 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321192.454162:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983261312:0@lo:400:kworker.0 00000100:00100000:1.0:1767321192.454209:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261952 00000100:00100000:1.0:1767321192.454212:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261952 00000100:00100000:2.0:1767321192.454650:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2167 00000100:00100000:2.0:1767321192.454657:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983261568:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321192.454717:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983261568:12345-0@lo:400:kworker.0 Request processed in 60us (867us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321192.454723:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2167 00010000:00080000:2.0:1767321192.454765:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8380 x1853163983261568/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321208 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321192.454775:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983261568:0@lo:400:kworker.0 00000100:00100000:2.0:1767321192.454813:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983261184:0@lo:250:kworker.0 00000400:00000100:2.0:1767321192.454834:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983261184 offset 0 length 520: 4 00000100:00100000:2.0:1767321192.454855:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983261440:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.454872:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.454886:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983261696:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.454903:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.454914:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983261824:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.454933:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.454963:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983262208:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.454983:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321192.455007:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261440 00000100:00100000:2.0:1767321192.455011:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261440 00000100:00100000:2.0:1767321192.455026:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261696 00000100:00100000:2.0:1767321192.455028:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261696 00000100:00100000:2.0:1767321192.455039:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262208 00000100:00100000:2.0:1767321192.455041:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262208 00000100:00100000:2.0:1767321192.455050:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2870 00000100:00100000:2.0:1767321192.455055:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983261440:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321192.455609:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321192.461792:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1331 00000100:00100000:3.0:1767321192.461812:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983262080:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321192.461915:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983262080:12345-0@lo:400:kworker.0 Request processed in 111us (8286us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321192.461923:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1331 00000100:00100000:3.0:1767321192.461989:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e56a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983262080:0@lo:400:kworker.0 00000100:00100000:3.0:1767321192.462047:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2871 00000100:00100000:3.0:1767321192.462053:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983261696:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321192.462074:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321192.463139:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983261696:12345-0@lo:38:kworker.0 Request processed in 1077us (8227us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321192.463151:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2871 00000100:00100000:3.0:1767321192.463187:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2872 00000100:00100000:3.0:1767321192.463194:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983262208:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321192.463211:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321192.463240:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382de680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983262208:12345-0@lo:38:kworker.0 Request processed in 45us (8256us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321192.463246:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2872 00000100:00080000:3.0:1767321192.463351:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321192.463357:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321192.463372:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983261696:0@lo:38:kworker.0 00000100:00080000:3.0:1767321192.463385:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321192.463389:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321192.463392:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e55180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983262208:0@lo:38:kworker.0 00000100:00100000:3.0:1767321192.463447:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983261824 00000100:00100000:3.0:1767321192.463454:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983261824 00000100:00100000:3.0:1767321192.463495:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 742 00000100:00100000:3.0:1767321192.463500:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983261824:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321192.463520:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321192.463554:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983261824:12345-0@lo:38:kworker.0 Request processed in 53us (8619us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321192.463561:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 742 00000100:00080000:3.0:1767321192.463587:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321192.463592:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321192.463595:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983261824:0@lo:38:kworker.0 00000100:00100000:1.0:1767321192.467176:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1332 00000100:00100000:1.0:1767321192.467193:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983261952:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321192.468016:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983261952:0@lo:400:kworker.0 00000100:00080000:0.0:1767321192.468112:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321192.468117:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321192.468121:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321192.468505:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435bb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983261952:12345-0@lo:400:kworker.0 Request processed in 115us (13383us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321192.468516:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1332 00000100:00080000:0.0:1767321192.480543:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321192.480550:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321192.480555:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983261440:0@lo:38:kworker.0 00000100:00100000:2.0:1767321192.482346:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ddc00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983261440:12345-0@lo:38:kworker.0 Request processed in 25329us (25512us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321192.482379:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2870 00010000:00010000:2.0:1767321192.580592:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321193.606911:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321194.382352:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321194.382701:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321194.382791:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724160 00000100:00100000:0.0:1767321194.382801:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724160 00000100:00100000:0.0:1767321194.382842:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724288 00000100:00100000:0.0:1767321194.382845:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724288 00000100:00100000:0.0:1767321194.382905:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2168 00000100:00100000:0.0:1767321194.382916:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724288:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321194.382997:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724288:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 84us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321194.383004:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2168 00000100:00100000:0.0:1767321194.383021:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2873 00000100:00100000:0.0:1767321194.383026:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915724160:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321194.383044:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321194.383079:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915724160:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 40us (831us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321194.383086:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2873 00000100:00080000:1.0:1767321194.383475:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321194.629765:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321194.948593:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983262336:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321194.948667:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321194.948776:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262336 00000100:00100000:0.0:1767321194.948785:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262336 00000100:00100000:0.0:1767321194.948857:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1444 00000100:00100000:0.0:1767321194.948867:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983262336:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1767321194.948895:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321194.948936:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983262336:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (269us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321194.948943:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1444 00000100:00100000:1.0:1767321194.949054:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934639e54380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983262336:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321194.949073:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321194.949323:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321194.949335:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983262464:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321194.949362:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321194.949424:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262464 00000100:00100000:1.0:1767321194.949428:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262464 00000100:00100000:1.0:1767321194.949474:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1445 00000100:00100000:1.0:1767321194.949480:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983262464:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321194.949497:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321194.949530:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e57800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983262464:12345-0@lo:13:osp-pre-0-1.0 Request processed in 49us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321194.949535:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1445 00000100:00100000:0.0:1767321194.949602:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983262464:0@lo:13:osp-pre-0-1.0 00000100:00080000:0.0:1767321194.949618:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321195.654126:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321196.676653:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321197.510197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.510207:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321197.510299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.510303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.510307:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321197.510314:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6440 00000100:00100000:3.0:1767321197.510316:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1767321197.510320:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.510327:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321197.510332:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983262592:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.510347:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359500 x1853163983262720/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321197.510392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.510395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.510397:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767321197.510397:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321197.510401:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6440 00000100:00080000:1.0:1767321197.510403:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.510406:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321197.510417:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359c00 x1853163983262976/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321197.510425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.510428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.510429:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321197.510432:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6440 00000100:00080000:1.0:1767321197.510434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.510437:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321197.510440:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262592 00000100:00100000:1.0:1767321197.510446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358380 x1853163983263104/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321197.510446:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262592 00000100:00080000:1.0:1767321197.510468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321197.510471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321197.510472:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321197.510475:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6440 00000100:00080000:1.0:1767321197.510477:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321197.510480:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321197.510492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359f80 x1853163983263488/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321197.510495:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169 00000100:00100000:3.0:1767321197.510503:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983262592:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.510512:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983263232:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.510542:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321197.510560:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983262592:12345-0@lo:400:kworker.0 Request processed in 58us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321197.510563:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321197.510568:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169 00000100:00100000:1.0:1767321197.510569:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983262848:0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.510587:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321197.510598:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262848 00000100:00100000:3.0:1767321197.510602:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262848 00000100:00100000:1.0:1767321197.510608:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263232 00000100:00100000:1.0:1767321197.510611:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263232 00000100:00100000:1.0:1767321197.510638:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1333 00000100:00100000:1.0:1767321197.510644:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983263232:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321197.510678:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983263232:12345-0@lo:400:kworker.0 Request processed in 34us (135us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321197.510684:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1333 00000100:00100000:2.0:1767321197.510726:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983262720:0@lo:38:kworker.0 00000100:00100000:1.0:1767321197.510731:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983263232:0@lo:400:kworker.0 00000100:00100000:2.0:1767321197.510754:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321197.510772:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983262976:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.510791:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321197.510814:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983263104:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.510836:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321197.510853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983263488:0@lo:38:kworker.0 00000100:00100000:2.0:1767321197.510871:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321197.510897:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262720 00000100:00100000:2.0:1767321197.510900:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262720 00000100:00100000:2.0:1767321197.510914:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983262976 00000100:00100000:2.0:1767321197.510916:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983262976 00000100:00100000:2.0:1767321197.510923:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263488 00000100:00100000:2.0:1767321197.510925:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263488 00000100:00100000:2.0:1767321197.510934:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2874 00000100:00100000:2.0:1767321197.510939:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983262720:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321197.510957:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321197.510963:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2875 00000100:00100000:1.0:1767321197.510967:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983262976:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321197.510977:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321197.511001:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983262976:12345-0@lo:38:kworker.0 Request processed in 31us (208us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321197.511001:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dd880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983262720:12345-0@lo:38:kworker.0 Request processed in 59us (244us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321197.511007:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2875 00000100:00100000:0.0:1767321197.511068:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1767321197.511078:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983263360:0@lo:400:kworker.0 00000100:00100000:0.0:1767321197.511108:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321197.511135:0:70997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263360 00000100:00100000:0.0:1767321197.511138:0:70997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263360 00000100:00100000:0.0:1767321197.511163:0:70997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1334 00000100:00100000:0.0:1767321197.511168:0:70997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983263360:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321197.511199:0:70997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983263360:12345-0@lo:400:kworker.0 Request processed in 31us (90us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321197.511204:0:70997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1334 00000100:00080000:0.0:1767321197.511221:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:0.0:1767321197.511224:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321197.511227:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321197.511245:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983263360:0@lo:400:kworker.0 00000100:00100000:2.0:1767321197.516542:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2874 00000100:00100000:1.0:1767321197.516577:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2876 00000100:00100000:1.0:1767321197.516590:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983263488:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321197.516704:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321197.516866:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983263488:12345-0@lo:38:kworker.0 Request processed in 278us (5992us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321197.516873:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2876 00000100:00100000:1.0:1767321197.516912:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263104 00000100:00100000:1.0:1767321197.516920:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263104 00000100:00100000:1.0:1767321197.516962:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 743 00000100:00100000:1.0:1767321197.516967:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983263104:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321197.516978:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321197.517007:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dea00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983263104:12345-0@lo:38:kworker.0 Request processed in 38us (6169us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321197.517014:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 743 00000100:00080000:2.0:1767321197.517161:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.517166:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00080000:1.0:1767321197.517170:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b100 x1853163983262592/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321213 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321197.517170:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983262720:0@lo:38:kworker.0 00000100:00100000:1.0:1767321197.517182:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983262592:0@lo:400:kworker.0 00000100:00080000:2.0:1767321197.517182:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.517186:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321197.517188:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983262976:0@lo:38:kworker.0 00000100:00080000:2.0:1767321197.517195:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.517198:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321197.517200:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983263104:0@lo:38:kworker.0 00000100:00080000:2.0:1767321197.517207:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321197.517210:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321197.517213:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983263488:0@lo:38:kworker.0 00000100:00100000:3.0:1767321197.517559:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2170 00000100:00100000:3.0:1767321197.517572:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983262848:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321197.517638:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435a300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7012:x1853163983262848:12345-0@lo:400:kworker.0 Request processed in 71us (7049us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321197.517645:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2170 00010000:00080000:2.0:1767321197.517903:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435aa00 x1853163983262848/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321213 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321197.517915:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:7012:1853163983262848:0@lo:400:kworker.0 00010000:00010000:2.0:1767321197.701785:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321198.728548:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321199.501048:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321199.501213:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321199.502758:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724544 00000100:00100000:0.0:1767321199.502772:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724544 00000100:00100000:0.0:1767321199.502883:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724416 00000100:00100000:0.0:1767321199.502887:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724416 00000100:00100000:0.0:1767321199.502913:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2171 00000100:00100000:0.0:1767321199.502922:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724544:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:0.0:1767321199.503001:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724544:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 80us (2652us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321199.503009:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2171 00000100:00100000:0.0:1767321199.503028:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2877 00000100:00100000:0.0:1767321199.503032:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915724416:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321199.503053:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321199.503078:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915724416:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (1865us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321199.503085:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2877 00000100:00080000:1.0:1767321199.511860:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321199.748568:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321200.069098:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321200.069122:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983263616:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321200.069191:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321200.069327:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263616 00000100:00100000:0.0:1767321200.069336:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263616 00000100:00100000:0.0:1767321200.069403:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1446 00000100:00100000:0.0:1767321200.069414:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983263616:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321200.069423:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00002000:00000024:0.0:1767321200.069447:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321200.069502:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983263616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (310us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321200.069509:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1446 00000100:00100000:1.0:1767321200.069967:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983263616:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321200.072641:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983263744:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321200.072716:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321200.072848:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263744 00000100:00100000:0.0:1767321200.072855:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263744 00000100:00100000:0.0:1767321200.072978:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1447 00000100:00100000:0.0:1767321200.072989:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983263744:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321200.073019:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321200.073074:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983263744:12345-0@lo:13:osp-pre-0-1.0 Request processed in 87us (356us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321200.073081:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1447 00000100:00100000:3.0:1767321200.073324:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346382dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983263744:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321200.073631:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:2.0:1767321200.775439:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321201.797318:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321202.628400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321202.628410:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321202.628471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321202.628475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321202.628479:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321202.628488:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6446 00000100:00080000:1.0:1767321202.628493:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321202.628500:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321202.628522:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358700 x1853163983264000/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321202.628579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321202.628582:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321202.628584:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321202.628588:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6446 00000100:00080000:1.0:1767321202.628591:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321202.628595:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321202.628612:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983264128:0@lo:400:kworker.0 00000100:00100000:1.0:1767321202.628688:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321202.628769:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604359f80 x1853163983264256/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321202.628778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321202.628781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321202.628784:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321202.628787:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6446 00000100:00080000:1.0:1767321202.628790:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321202.628793:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321202.628803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358380 x1853163983264384/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321202.628828:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983264512:0@lo:400:kworker.0 00000100:00100000:1.0:1767321202.628856:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321202.628891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321202.628895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321202.628897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321202.628900:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6446 00000100:00080000:1.0:1767321202.628903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321202.628906:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321202.628917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076faa00 x1853163983264768/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321202.629043:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983263872:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.629080:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.629103:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1767321202.629108:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983264640:0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.629142:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321202.629205:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264128 00000100:00100000:2.0:1767321202.629213:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264128 00000100:00100000:2.0:1767321202.629288:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983263872 00000100:00100000:2.0:1767321202.629292:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983263872 00000100:00100000:2.0:1767321202.629316:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172 00000100:00100000:2.0:1767321202.629325:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983264128:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.629382:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435aa00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983264128:12345-0@lo:400:kworker.0 Request processed in 56us (693us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321202.629389:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172 00000100:00100000:2.0:1767321202.629404:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2173 00000100:00100000:2.0:1767321202.629410:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983263872:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321202.629438:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983263872:12345-0@lo:400:kworker.0 Request processed in 28us (362us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321202.629444:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2173 00000100:00080000:2.0:1767321202.629460:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:2.0:1767321202.629484:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b100 x1853163983263872/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321218 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321202.629496:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983263872:0@lo:400:kworker.0 00010000:00080000:2.0:1767321202.629524:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435ad80 x1853163983264128/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321218 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321202.629533:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983264128:0@lo:400:kworker.0 00000100:00100000:0.0:1767321202.629665:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264512 00000100:00100000:0.0:1767321202.629670:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264512 00000100:00100000:0.0:1767321202.629710:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264640 00000100:00100000:0.0:1767321202.629713:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264640 00000100:00100000:0.0:1767321202.629734:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1335 00000100:00100000:0.0:1767321202.629740:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983264512:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321202.629780:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983264512:12345-0@lo:400:kworker.0 Request processed in 40us (923us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321202.629787:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1335 00000100:00100000:0.0:1767321202.629800:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1336 00000100:00100000:0.0:1767321202.629806:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983264640:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321202.629835:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382df100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983264640:12345-0@lo:400:kworker.0 Request processed in 30us (692us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321202.629841:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1336 00000100:00100000:0.0:1767321202.629876:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983264000:0@lo:38:kworker.0 00000100:00100000:0.0:1767321202.629905:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321202.629923:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983264256:0@lo:38:kworker.0 00000100:00100000:0.0:1767321202.629942:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321202.629956:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983264384:0@lo:38:kworker.0 00000100:00100000:0.0:1767321202.629998:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321202.630015:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983264768:0@lo:38:kworker.0 00000100:00100000:0.0:1767321202.630036:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321202.630061:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264384 00000100:00100000:0.0:1767321202.630065:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264384 00000100:00100000:0.0:1767321202.630084:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 744 00000100:00100000:0.0:1767321202.630090:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983264384:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321202.630109:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321202.630140:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e56a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983264384:12345-0@lo:38:kworker.0 Request processed in 49us (142us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321202.630146:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 744 00000100:00100000:0.0:1767321202.630163:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264000 00000100:00100000:0.0:1767321202.630166:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264000 00000100:00100000:0.0:1767321202.630191:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264256 00000100:00100000:0.0:1767321202.630194:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264256 00000100:00100000:0.0:1767321202.630201:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264768 00000100:00100000:0.0:1767321202.630203:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264768 00000100:00100000:0.0:1767321202.630211:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1767321202.630216:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264000:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321202.630225:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321202.630256:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e54380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264000:12345-0@lo:38:kworker.0 Request processed in 39us (350us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321202.630262:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1767321202.630307:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2879 00000100:00100000:0.0:1767321202.630312:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264256:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321202.630321:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321202.630343:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639e55180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264256:12345-0@lo:38:kworker.0 Request processed in 29us (399us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321202.630348:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2879 00000100:00100000:0.0:1767321202.630357:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2880 00000100:00100000:0.0:1767321202.630361:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264768:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321202.630371:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321202.630399:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983264768:12345-0@lo:38:kworker.0 Request processed in 36us (362us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321202.630404:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2880 00000100:00100000:0.0:1767321202.630437:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983264512:0@lo:400:kworker.0 00000100:00080000:0.0:1767321202.630462:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321202.630467:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321202.630470:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983264000:0@lo:38:kworker.0 00000100:00080000:0.0:1767321202.630478:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321202.630481:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321202.630484:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604359f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983264256:0@lo:38:kworker.0 00000100:00080000:0.0:1767321202.630492:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321202.630495:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321202.630497:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983264384:0@lo:38:kworker.0 00000100:00080000:0.0:1767321202.630504:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321202.630507:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321202.630510:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983264768:0@lo:38:kworker.0 00000100:00080000:2.0:1767321202.631199:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321202.631206:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321202.631237:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983264640:0@lo:400:kworker.0 00010000:00010000:3.0:1767321202.823608:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321203.844545:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321204.620824:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321204.621308:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321204.621858:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724800 00000100:00100000:2.0:1767321204.621870:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724800 00000100:00100000:2.0:1767321204.621945:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724672 00000100:00100000:2.0:1767321204.621949:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724672 00000100:00100000:2.0:1767321204.621970:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2174 00000100:00100000:2.0:1767321204.621979:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724800:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321204.622055:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915724800:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 76us (1240us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321204.622063:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2174 00000100:00100000:2.0:1767321204.622093:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2881 00000100:00100000:2.0:1767321204.622098:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915724672:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321204.622118:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321204.622156:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915724672:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 55us (848us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321204.622167:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2881 00000100:00080000:0.0:1767321204.623517:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321204.870645:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321205.189241:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321205.189295:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983264896:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321205.189381:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321205.189604:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983264896 00000100:00100000:1.0:1767321205.189613:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983264896 00000100:00100000:1.0:1767321205.189676:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1448 00000100:00100000:1.0:1767321205.189685:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604b86850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983264896:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321205.189717:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321205.189765:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604b86850 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983264896:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321205.189772:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1448 00000100:00100000:1.0:1767321205.190022:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983264896:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1767321205.190687:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321205.191137:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983265024:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321205.191184:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321205.191848:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265024 00000100:00100000:0.0:1767321205.191856:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265024 00000100:00100000:0.0:1767321205.191930:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1449 00000100:00100000:0.0:1767321205.191940:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983265024:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321205.191968:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321205.192017:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382dc000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983265024:12345-0@lo:13:osp-pre-0-1.0 Request processed in 79us (831us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321205.192024:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1449 00000100:00100000:2.0:1767321205.192117:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983265024:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321205.192202:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767321205.892512:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321206.917702:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1767321207.748440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321207.748452:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1767321207.748502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321207.748505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321207.748509:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321207.748517:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6451 00000100:00080000:1.0:1767321207.748522:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321207.748529:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321207.748549:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076fa680 x1853163983265280/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321207.748593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321207.748597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321207.748598:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321207.748602:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6451 00000100:00080000:1.0:1767321207.748605:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321207.748608:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321207.748619:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8380 x1853163983265536/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321207.748627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321207.748630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321207.748632:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1767321207.748635:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6451 00000100:00080000:1.0:1767321207.748637:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321207.748641:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321207.748651:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346076f8000 x1853163983265664/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1767321207.748673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1767321207.748677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1767321207.748679:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1767321207.748763:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983265280:0@lo:38:kworker.0 00000100:00080000:1.0:1767321207.748776:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6451 00000100:00080000:1.0:1767321207.748778:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1767321207.748782:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:1.0:1767321207.748794:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934612b14a80 x1853163983266048/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321207.748831:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321207.748890:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983265536:0@lo:38:kworker.0 00000100:00100000:1.0:1767321207.748901:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983265408:0@lo:400:kworker.0 00000100:00100000:3.0:1767321207.748913:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321207.748924:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321207.748939:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983265664:0@lo:38:kworker.0 00000100:00100000:1.0:1767321207.748943:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321207.748947:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983265792:0@lo:400:kworker.0 00000100:00100000:3.0:1767321207.748962:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321207.748966:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321207.748987:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934612b14a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983266048:0@lo:38:kworker.0 00000100:00100000:3.0:1767321207.749008:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321207.749033:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265792 00000100:00100000:3.0:1767321207.749039:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265792 00000100:00100000:3.0:1767321207.749104:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1337 00000100:00100000:3.0:1767321207.749113:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983265792:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321207.749162:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983265792:12345-0@lo:400:kworker.0 Request processed in 50us (194us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321207.749169:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1337 00000100:00100000:3.0:1767321207.749193:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265280 00000100:00100000:3.0:1767321207.749196:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265280 00000100:00100000:3.0:1767321207.749212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265536 00000100:00100000:3.0:1767321207.749214:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265536 00000100:00100000:3.0:1767321207.749221:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265408 00000100:00100000:3.0:1767321207.749222:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265408 00000100:00100000:3.0:1767321207.749236:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266048 00000100:00100000:3.0:1767321207.749239:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266048 00000100:00100000:3.0:1767321207.749246:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2175 00000100:00100000:3.0:1767321207.749251:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983265408:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321207.749305:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983265408:12345-0@lo:400:kworker.0 Request processed in 53us (379us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321207.749311:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2175 00000100:00100000:3.0:1767321207.749323:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2882 00000100:00100000:3.0:1767321207.749326:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983265280:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321207.749344:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321207.749372:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265664 00000100:00100000:3.0:1767321207.749373:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d92300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983265280:12345-0@lo:38:kworker.0 Request processed in 45us (542us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321207.749376:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265664 00000100:00100000:3.0:1767321207.749379:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2882 00000100:00100000:3.0:1767321207.749389:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2883 00000100:00100000:3.0:1767321207.749394:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983265536:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321207.749396:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 745 00000100:00100000:1.0:1767321207.749399:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983265664:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321207.749402:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00020000:1.0:1767321207.749407:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321207.749424:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d90a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983265536:12345-0@lo:38:kworker.0 Request processed in 29us (510us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321207.749426:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934639d93b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983265664:12345-0@lo:38:kworker.0 Request processed in 26us (464us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321207.749430:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1767321207.749431:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 745 00000100:00100000:3.0:1767321207.749440:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2884 00000100:00100000:3.0:1767321207.749444:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983266048:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321207.749452:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767321207.749462:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321207.749465:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321207.749471:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983266048:12345-0@lo:38:kworker.0 Request processed in 26us (463us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321207.749477:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2884 00000100:00080000:3.0:1767321207.749495:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1767321207.749496:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076faa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983265792:0@lo:400:kworker.0 00000100:00080000:3.0:1767321207.749499:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321207.749502:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934612b14a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983266048:0@lo:38:kworker.0 00010000:00080000:1.0:1767321207.749506:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076f8700 x1853163983265408/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321223 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1767321207.749508:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321207.749511:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321207.749514:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983265280:0@lo:38:kworker.0 00000100:00100000:1.0:1767321207.749515:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983265408:0@lo:400:kworker.0 00000100:00080000:3.0:1767321207.749520:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321207.749522:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321207.749524:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983265536:0@lo:38:kworker.0 00000100:00100000:2.0:1767321207.749668:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983265920:0@lo:400:kworker.0 00000100:00100000:2.0:1767321207.749707:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321207.749735:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321207.749741:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983265152:0@lo:400:kworker.0 00000100:00100000:2.0:1767321207.749762:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321207.749791:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265152 00000100:00100000:2.0:1767321207.749794:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265152 00000100:00080000:3.0:1767321207.749962:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321207.749967:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321207.749970:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983265664:0@lo:38:kworker.0 00000100:00100000:2.0:1767321207.749986:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2176 00000100:00100000:2.0:1767321207.749993:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983265152:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321207.750005:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:3.0:1767321207.750031:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346076fa300 x1853163983265152/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321223 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321207.750041:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076fa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0000_UUID:7013:1853163983265152:0@lo:400:kworker.0 00000100:00100000:2.0:1767321207.750252:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7013:x1853163983265152:12345-0@lo:400:kworker.0 Request processed in 32us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321207.750260:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2176 00000100:00100000:1.0:1767321207.750418:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983265920 00000100:00100000:1.0:1767321207.750422:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983265920 00000100:00100000:1.0:1767321207.750468:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1338 00000100:00100000:1.0:1767321207.750475:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983265920:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321207.750496:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321207.750510:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346382ded80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983265920:12345-0@lo:400:kworker.0 Request processed in 36us (802us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321207.750516:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1338 00000100:00100000:3.0:1767321207.750517:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983265920:0@lo:400:kworker.0 00010000:00010000:2.0:1767321207.940514:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321208.965026:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321209.740798:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321209.740995:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725056 00000100:00100000:2.0:1767321209.741008:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725056 00000100:00100000:2.0:1767321209.741176:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2177 00000100:00100000:2.0:1767321209.741187:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915725056:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321209.741314:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915725056:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 129us (525us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321209.741326:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2177 00000100:00080000:3.0:1767321209.742011:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321209.745710:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321209.746139:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915724928 00000100:00100000:0.0:1767321209.746149:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915724928 00000100:00100000:0.0:1767321209.746213:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2885 00000100:00100000:0.0:1767321209.746221:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b15f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915724928:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321209.746245:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321209.746353:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b15f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915724928:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 131us (655us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321209.746361:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2885 00010000:00010000:2.0:1767321209.988640:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321210.308592:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934612b14e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983266176:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321210.308664:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321210.308685:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266176 00000100:00100000:2.0:1767321210.308694:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266176 00000100:00100000:2.0:1767321210.308744:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1450 00000100:00100000:2.0:1767321210.308753:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983266176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321210.308782:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321210.308834:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983266176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (169us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321210.308841:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1450 00000100:00100000:3.0:1767321210.308901:0:7012:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1767321210.308914:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983266304:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321210.308942:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1767321210.308974:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:2.0:1767321210.309062:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266304 00000100:00100000:2.0:1767321210.309066:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266304 00000100:00100000:0.0:1767321210.309080:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934612b14e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983266176:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321210.309103:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1451 00000100:00100000:2.0:1767321210.309110:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983266304:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321210.309126:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321210.309164:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983266304:12345-0@lo:13:osp-pre-0-1.0 Request processed in 55us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321210.309170:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1451 00000100:00080000:3.0:1767321210.309326:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321210.309349:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983266304:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321211.015215:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321212.036509:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321212.868653:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321212.868663:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1767321212.868742:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1767321212.870179:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321212.870186:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321212.870190:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321212.870198:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6456 00000100:00080000:0.0:1767321212.870203:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321212.870210:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321212.870247:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014a80 x1853163983266560/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321212.870325:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321212.870330:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321212.870332:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321212.870336:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6456 00000100:00080000:0.0:1767321212.870338:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321212.870342:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321212.870356:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014380 x1853163983266816/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321212.870366:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321212.870368:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321212.870370:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321212.870373:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6456 00000100:00080000:0.0:1767321212.870376:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321212.870379:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321212.870391:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163983266944/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321212.870469:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983266688:0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.870534:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321212.870609:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1767321212.870616:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983267072:0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.870642:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321212.870679:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267072 00000100:00100000:2.0:1767321212.870686:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267072 00000100:00100000:2.0:1767321212.870740:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1339 00000100:00100000:2.0:1767321212.870748:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983267072:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.870799:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983267072:12345-0@lo:400:kworker.0 Request processed in 51us (156us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321212.870806:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1339 00000100:00100000:2.0:1767321212.870848:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983267072:0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.870869:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266688 00000100:00100000:2.0:1767321212.870872:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266688 00000100:00100000:2.0:1767321212.870898:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178 00000100:00100000:2.0:1767321212.870904:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983266688:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.870936:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983266688:12345-0@lo:400:kworker.0 Request processed in 32us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321212.870943:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178 00010000:00080000:2.0:1767321212.870965:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014e00 x1853163983266688/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321228 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321212.870974:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983266688:0@lo:400:kworker.0 00000100:00080000:3.0:1767321212.871355:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:3.0:1767321212.871361:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321212.871375:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983266560:0@lo:38:kworker.0 00000100:00100000:3.0:1767321212.871408:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321212.871441:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983266816:0@lo:38:kworker.0 00000100:00100000:3.0:1767321212.871460:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321212.871476:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983266944:0@lo:38:kworker.0 00000100:00100000:3.0:1767321212.871496:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321212.871531:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266944 00000100:00100000:3.0:1767321212.871535:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266944 00000100:00100000:3.0:1767321212.871570:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 746 00000100:00100000:3.0:1767321212.871575:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983266944:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321212.871594:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321212.871626:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983266944:12345-0@lo:38:kworker.0 Request processed in 50us (129us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321212.871633:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 746 00000100:00080000:3.0:1767321212.871661:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321212.871665:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321212.871668:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983266944:0@lo:38:kworker.0 00000100:00100000:3.0:1767321212.871684:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266560 00000100:00100000:3.0:1767321212.871687:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266560 00000100:00100000:3.0:1767321212.871701:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266816 00000100:00100000:3.0:1767321212.871704:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266816 00000100:00100000:3.0:1767321212.871716:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2886 00000100:00100000:3.0:1767321212.871721:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983266560:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321212.871730:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321212.871755:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983266560:12345-0@lo:38:kworker.0 Request processed in 34us (347us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321212.871761:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2886 00000100:00100000:3.0:1767321212.871771:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2887 00000100:00100000:3.0:1767321212.871775:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983266816:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321212.871783:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321212.871803:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983266816:12345-0@lo:38:kworker.0 Request processed in 27us (343us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321212.871808:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2887 00000100:00080000:3.0:1767321212.871835:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321212.871839:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321212.871843:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983266560:0@lo:38:kworker.0 00000100:00080000:3.0:1767321212.871850:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321212.871853:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321212.871856:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983266816:0@lo:38:kworker.0 00000100:00100000:1.0:1767321212.876842:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983267200:0@lo:400:kworker.0 00000100:00100000:1.0:1767321212.877518:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321212.877580:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983266432:0@lo:400:kworker.0 00000100:00100000:1.0:1767321212.877611:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321212.877775:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267200 00000100:00100000:2.0:1767321212.877799:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267200 00000100:00100000:2.0:1767321212.877865:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1340 00000100:00100000:2.0:1767321212.877874:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983267200:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321212.877929:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983267200:12345-0@lo:400:kworker.0 Request processed in 57us (1008us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321212.877948:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1340 00000100:00100000:3.0:1767321212.878226:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983266432 00000100:00100000:3.0:1767321212.878232:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983266432 00000100:00100000:3.0:1767321212.878293:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2179 00000100:00100000:3.0:1767321212.878302:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934612b14a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983266432:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321212.878336:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934612b14a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983266432:12345-0@lo:400:kworker.0 Request processed in 37us (725us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321212.878342:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2179 00000100:00080000:1.0:1767321212.878459:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00080000:1.0:1767321212.878465:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321212.878499:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983267200:0@lo:400:kworker.0 00010000:00080000:1.0:1767321212.878513:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015880 x1853163983266432/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321228 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321212.878524:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983266432:0@lo:400:kworker.0 00000100:00080000:0.0:1767321212.879233:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321212.879239:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321212.879243:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321212.879248:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6456 00000100:00080000:0.0:1767321212.879252:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321212.879259:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321212.879303:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017480 x1853163983267328/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321212.879402:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983267328:0@lo:38:kworker.0 00000100:00100000:2.0:1767321212.879447:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321212.879647:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267328 00000100:00100000:0.0:1767321212.879654:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267328 00000100:00100000:0.0:1767321212.879688:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2888 00000100:00100000:0.0:1767321212.879694:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983267328:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321212.879713:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321212.879757:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983267328:12345-0@lo:38:kworker.0 Request processed in 62us (309us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321212.879764:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2888 00000100:00080000:2.0:1767321212.879871:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321212.879875:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321212.879878:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983267328:0@lo:38:kworker.0 00010000:00010000:2.0:1767321213.062929:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321213.842766:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321213.842931:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725440 00000100:00100000:2.0:1767321213.842941:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725440 00000100:00100000:2.0:1767321213.843044:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2180 00000100:00100000:2.0:1767321213.843055:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915725440:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321213.843136:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:ee100a64-1c8d-4519-95f7-836c165fda80+6:2508:x1853163915725440:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 83us (379us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321213.843144:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2180 00000100:00080000:1.0:1767321213.843232:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000400:00000100:1.0:1767321213.858128:0:7006:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915725184 offset 0 length 520: 4 00000100:00100000:1.0:1767321213.858398:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321213.858490:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725312 00000100:00100000:0.0:1767321213.858501:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725312 00000100:00100000:0.0:1767321213.858559:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2889 00000100:00100000:0.0:1767321213.858567:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915725312:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:0.0:1767321213.858589:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321213.858658:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915725312:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 90us (260us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321213.858667:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2889 00010000:00010000:2.0:1767321214.086510:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321215.108584:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321215.428625:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321215.428642:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983267456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321215.428708:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321215.428816:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267456 00000100:00100000:1.0:1767321215.428824:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267456 00000100:00100000:1.0:1767321215.428889:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1452 00000100:00100000:1.0:1767321215.428898:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983267456:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321215.428928:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321215.428982:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983267456:12345-0@lo:13:osp-pre-0-1.0 Request processed in 85us (273us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321215.428989:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1452 00000100:00080000:2.0:1767321215.429055:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321215.429094:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983267456:0@lo:13:osp-pre-0-1.0 00000100:00100000:2.0:1767321215.429306:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983267584:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321215.429333:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321215.433377:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267584 00000100:00100000:3.0:1767321215.433394:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267584 00000100:00100000:3.0:1767321215.433467:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1453 00000100:00100000:3.0:1767321215.433478:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983267584:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321215.433514:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:1.0:1767321215.435056:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321215.444178:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983267584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 10693us (14834us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321215.444194:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1453 00000100:00100000:2.0:1767321215.454098:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983267584:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1767321216.132529:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321217.156591:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321217.988401:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321192/real 1767321192] req@ffff9346076f9180 x1853163983261184/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321217 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321217.988433:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9346076f9180 x1853163983261184/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321217 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1767321217.988454:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.988473:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:2.0:1767321217.988477:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346076f9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983261184:0@lo:250:kworker.0 00000100:00080000:0.0:1767321217.989432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.989440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1767321217.989443:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.989458:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6440 00000100:00080000:0.0:1767321217.989462:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.989470:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321217.989507:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017800 x1853163983267712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.989567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.989571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.989573:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.989576:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.989579:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.989582:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321217.989597:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633014000 x1853163983267968/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.989619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.989621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.989623:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.989626:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.989628:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.989631:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321217.989640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015880 x1853163983268224/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.989648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.989650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.989652:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.989655:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.989657:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.989660:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321217.989672:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015500 x1853163983268352/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321217.989692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321217.989695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321217.989697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321217.989700:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6461 00000100:00080000:0.0:1767321217.989702:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321217.989705:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321217.989715:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633017b80 x1853163983268736/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321217.989822:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983268096:0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.989903:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.989937:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983267840:0@lo:400:kworker.0 00000100:00100000:1.0:1767321217.989963:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321217.989967:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1767321217.989972:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983268480:0@lo:400:kworker.0 00000100:00100000:1.0:1767321217.989992:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321217.989995:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.989997:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983268608:0@lo:400:kworker.0 00000100:00100000:1.0:1767321217.990016:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.990044:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268096 00000100:00100000:1.0:1767321217.990051:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268096 00000100:00100000:3.0:1767321217.990077:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983267712:0@lo:250:kworker.0 00000100:00100000:1.0:1767321217.990096:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267840 00000400:00000100:3.0:1767321217.990098:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983267712 offset 0 length 520: 4 00000100:00100000:1.0:1767321217.990099:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267840 00000100:00100000:3.0:1767321217.990115:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983267968:0@lo:38:kworker.0 00000100:00100000:1.0:1767321217.990134:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181 00000100:00100000:3.0:1767321217.990136:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.990143:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983268096:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321217.990152:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983268224:0@lo:38:kworker.0 00000100:00100000:3.0:1767321217.990168:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321217.990177:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983268352:0@lo:38:kworker.0 00000100:00100000:1.0:1767321217.990192:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983268096:12345-0@lo:400:kworker.0 Request processed in 50us (289us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321217.990197:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.990199:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181 00000100:00100000:3.0:1767321217.990210:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983268736:0@lo:38:kworker.0 00000100:00100000:1.0:1767321217.990212:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983267968 00000100:00100000:1.0:1767321217.990215:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983267968 00000100:00100000:3.0:1767321217.990227:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321217.990237:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268224 00000100:00100000:1.0:1767321217.990240:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268224 00000100:00100000:3.0:1767321217.990246:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268736 00000100:00100000:3.0:1767321217.990249:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268736 00000100:00100000:1.0:1767321217.990250:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2182 00000100:00100000:1.0:1767321217.990255:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983267840:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321217.990261:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2890 00000100:00100000:3.0:1767321217.990265:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983267968:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321217.990310:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983267840:12345-0@lo:400:kworker.0 Request processed in 54us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321217.990316:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2182 00000100:00100000:1.0:1767321217.990328:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2891 00000100:00100000:1.0:1767321217.990331:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983268224:12345-0@lo:38:kworker.0 00000100:00100000:0.0:1767321217.990431:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2892 00000100:00100000:0.0:1767321217.990437:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983268736:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.990476:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268480 00000100:00100000:2.0:1767321217.990480:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268480 00000100:00100000:2.0:1767321217.990509:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268608 00000100:00100000:2.0:1767321217.990511:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268608 00000100:00100000:2.0:1767321217.990532:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1341 00000100:00100000:2.0:1767321217.990538:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983268480:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.990569:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604359c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983268480:12345-0@lo:400:kworker.0 Request processed in 31us (573us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321217.990575:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1341 00000100:00100000:2.0:1767321217.990586:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1342 00000100:00100000:2.0:1767321217.990592:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983268608:12345-0@lo:400:kworker.0 00010000:00020000:3.0:1767321217.990623:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321217.990655:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983267968:12345-0@lo:38:kworker.0 Request processed in 389us (518us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321217.990661:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2890 00000100:00100000:3.0:1767321217.990679:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268352 00000100:00100000:3.0:1767321217.990682:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268352 00000100:00100000:3.0:1767321217.990700:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 747 00000100:00100000:3.0:1767321217.990704:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983268352:12345-0@lo:38:kworker.0 00000100:00100000:2.0:1767321217.990734:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983268608:12345-0@lo:400:kworker.0 Request processed in 142us (717us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321217.990739:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1342 00000100:00080000:2.0:1767321217.990791:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.990796:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.990801:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983267968:0@lo:38:kworker.0 00000100:00080000:2.0:1767321217.990845:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:2.0:1767321217.990849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:2.0:1767321217.990851:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:2.0:1767321217.990882:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983268480:0@lo:400:kworker.0 00010000:00080000:2.0:1767321217.990905:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633017480 x1853163983268096/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321233 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321217.990913:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983268096:0@lo:400:kworker.0 00000100:00100000:2.0:1767321217.990947:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983268608:0@lo:400:kworker.0 00010000:00080000:2.0:1767321217.990956:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633016680 x1853163983267840/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321233 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321217.990963:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983267840:0@lo:400:kworker.0 00010000:00020000:1.0:1767321217.991076:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321217.991164:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983268224:12345-0@lo:38:kworker.0 Request processed in 830us (993us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321217.991170:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2891 00000100:00080000:2.0:1767321217.991350:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.991355:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00010000:00020000:0.0:1767321217.994061:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321217.994142:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983268736:12345-0@lo:38:kworker.0 Request processed in 3701us (3911us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321217.994154:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2892 00010000:00020000:3.0:1767321217.995416:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321217.995417:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983268224:0@lo:38:kworker.0 00000100:00080000:2.0:1767321217.995451:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.995456:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.995459:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983268736:0@lo:38:kworker.0 00000100:00100000:3.0:1767321217.996003:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e0a80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983268352:12345-0@lo:38:kworker.0 Request processed in 5293us (5801us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321217.996013:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 747 00000100:00080000:2.0:1767321217.996165:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321217.996169:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321217.996173:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983268352:0@lo:38:kworker.0 00010000:00010000:3.0:1767321218.183460:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321218.955973:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321218.956056:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321218.956157:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725696 00000100:00100000:2.0:1767321218.956167:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725696 00000100:00100000:2.0:1767321218.956228:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725568 00000100:00100000:2.0:1767321218.956233:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725568 00000100:00100000:2.0:1767321218.956252:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2183 00000100:00100000:2.0:1767321218.956262:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915725696:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:2.0:1767321218.956377:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915725696:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 115us (414us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321218.956385:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2183 00000100:00100000:2.0:1767321218.956403:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2893 00000100:00100000:2.0:1767321218.956407:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915725568:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:2.0:1767321218.956427:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321218.956453:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:2507:x1853163915725568:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (397us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321218.956460:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2893 00000100:00080000:3.0:1767321218.956714:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321219.204522:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321220.228395:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321220.549109:0:7013:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1767321220.549132:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983268864:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321220.549201:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321220.549308:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268864 00000100:00100000:0.0:1767321220.549317:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268864 00000100:00100000:0.0:1767321220.549378:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1454 00000100:00100000:0.0:1767321220.549387:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983268864:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:0.0:1767321220.549417:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:0.0:1767321220.549463:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076fa680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983268864:12345-0@lo:13:osp-pre-0-1.0 Request processed in 76us (261us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321220.549471:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1454 00000100:00100000:0.0:1767321220.549519:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983268864:0@lo:13:osp-pre-0-1.0 00000100:00080000:1.0:1767321220.549592:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321220.549965:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983268992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1767321220.550000:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321220.550141:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983268992 00000100:00100000:1.0:1767321220.550146:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983268992 00000100:00100000:1.0:1767321220.550196:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1455 00000100:00100000:1.0:1767321220.550203:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983268992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321220.550221:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00080000:3.0:1767321220.550298:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321220.550398:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435ad80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983268992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 196us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321220.550404:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1455 00000100:00100000:2.0:1767321220.550408:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983268992:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1767321221.253370:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321222.276876:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1767321223.109961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321223.109973:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1767321223.111381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321223.111386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321223.111390:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321223.111398:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6466 00000100:00080000:0.0:1767321223.111402:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321223.111408:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321223.111428:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016300 x1853163983269248/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321223.111455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321223.111458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321223.111459:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321223.111476:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6466 00000100:00080000:0.0:1767321223.111478:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321223.111481:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321223.111490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015c00 x1853163983269504/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321223.111497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321223.111500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321223.111501:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321223.111504:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6466 00000100:00080000:0.0:1767321223.111506:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321223.111509:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321223.111517:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633015500 x1853163983269632/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321223.111542:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1767321223.111550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1767321223.111552:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1767321223.111555:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6466 00000100:00080000:0.0:1767321223.111557:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1767321223.111561:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321223.111571:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934633016680 x1853163983270016/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321223.111601:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983269376:0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.111605:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983269888:0@lo:400:kworker.0 00000100:00100000:2.0:1767321223.111676:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983269248:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.111682:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983269120:0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.114256:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321223.114383:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983269760:0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.114413:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321223.114470:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269888 00000100:00100000:1.0:1767321223.114477:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269888 00000100:00100000:1.0:1767321223.114526:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269760 00000100:00100000:1.0:1767321223.114530:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269760 00000100:00100000:1.0:1767321223.114567:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1343 00000100:00100000:1.0:1767321223.114576:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983269888:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.114626:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f8700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983269888:12345-0@lo:400:kworker.0 Request processed in 52us (377us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321223.114633:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1343 00000100:00100000:1.0:1767321223.114650:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1344 00000100:00100000:1.0:1767321223.114655:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983269760:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.114683:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076faa00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983269760:12345-0@lo:400:kworker.0 Request processed in 28us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321223.114688:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1344 00000100:00100000:1.0:1767321223.114728:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983269888:0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.114749:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983269760:0@lo:400:kworker.0 00000100:00080000:1.0:1767321223.114789:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:1.0:1767321223.114808:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321223.119562:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321223.119717:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321223.119749:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321223.119774:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983269504:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.119778:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269376 00000100:00100000:3.0:1767321223.119786:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269376 00000100:00100000:1.0:1767321223.119821:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321223.119848:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269120 00000100:00100000:3.0:1767321223.119851:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269120 00000100:00100000:1.0:1767321223.119853:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983269632:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.119866:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269248 00000100:00100000:3.0:1767321223.119868:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269248 00000100:00100000:1.0:1767321223.119876:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321223.119876:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269504 00000100:00100000:3.0:1767321223.119878:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269504 00000100:00100000:3.0:1767321223.119888:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184 00000100:00100000:1.0:1767321223.119893:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983270016:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.119896:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983269376:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.119911:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1767321223.119943:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321223.119944:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7013:x1853163983269376:12345-0@lo:400:kworker.0 Request processed in 48us (391us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321223.119950:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184 00000100:00100000:1.0:1767321223.119955:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270016 00000100:00100000:1.0:1767321223.119958:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270016 00000100:00100000:3.0:1767321223.119966:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2185 00000100:00100000:3.0:1767321223.119971:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983269120:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321223.119974:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2894 00000100:00100000:1.0:1767321223.119978:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983269248:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321223.119995:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321223.119997:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983269120:12345-0@lo:400:kworker.0 Request processed in 26us (280us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321223.120002:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2185 00000100:00100000:3.0:1767321223.120012:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2895 00000100:00100000:3.0:1767321223.120016:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983269504:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321223.120024:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346076f9180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983269248:12345-0@lo:38:kworker.0 Request processed in 45us (275us total) trans 0 rc -19/-19 00010000:00020000:3.0:1767321223.120026:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321223.120030:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2894 00000100:00100000:1.0:1767321223.120042:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2896 00000100:00100000:3.0:1767321223.120043:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ea00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983269504:12345-0@lo:38:kworker.0 Request processed in 27us (223us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321223.120048:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2895 00000100:00100000:1.0:1767321223.120057:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983270016:12345-0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.120064:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983269632 00010000:00020000:1.0:1767321223.120066:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321223.120067:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983269632 00000100:00100000:3.0:1767321223.120084:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 748 00000100:00100000:1.0:1767321223.120087:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398e680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983270016:12345-0@lo:38:kworker.0 Request processed in 41us (176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321223.120088:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983269632:12345-0@lo:38:kworker.0 00000100:00100000:1.0:1767321223.120093:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2896 00010000:00020000:3.0:1767321223.120097:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1767321223.120126:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1767321223.120130:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983269632:12345-0@lo:38:kworker.0 Request processed in 40us (252us total) trans 0 rc -19/-19 00000100:00080000:1.0:1767321223.120132:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321223.120135:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983269248:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.120135:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 748 00000100:00080000:1.0:1767321223.120145:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321223.120148:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321223.120151:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983269504:0@lo:38:kworker.0 00000100:00080000:1.0:1767321223.120159:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00010000:00080000:3.0:1767321223.120159:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633014e00 x1853163983269120/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321239 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321223.120161:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321223.120164:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983269632:0@lo:38:kworker.0 00000100:00100000:3.0:1767321223.120170:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633014e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983269120:0@lo:400:kworker.0 00000100:00080000:1.0:1767321223.120171:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321223.120174:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321223.120176:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633016680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983270016:0@lo:38:kworker.0 00010000:00080000:0.0:1767321223.120314:0:7013:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff934633015180 x1853163983269376/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321239 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321223.120324:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633015180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:7013:1853163983269376:0@lo:400:kworker.0 00010000:00010000:2.0:1767321223.301398:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321224.076576:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321224.076794:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321224.082330:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725952 00000100:00100000:2.0:1767321224.082344:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725952 00000100:00100000:2.0:1767321224.082473:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915725824 00000100:00100000:2.0:1767321224.082479:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915725824 00000100:00100000:2.0:1767321224.082510:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2186 00000100:00100000:2.0:1767321224.082521:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915725952:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321224.082729:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2897 00000100:00100000:3.0:1767321224.082736:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915725824:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321224.082756:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321224.082836:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915725824:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 98us (6041us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321224.082843:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2897 00000100:00100000:2.0:1767321224.083124:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e1880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915725952:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 604us (6559us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321224.083132:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2186 00000100:00080000:1.0:1767321224.083305:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:3.0:1767321224.325681:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321225.349453:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321225.668659:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1767321225.668682:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983270144:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321225.668757:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321225.668911:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270144 00000100:00100000:2.0:1767321225.668918:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270144 00000100:00100000:2.0:1767321225.668984:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1456 00000100:00100000:2.0:1767321225.668993:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983270144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321225.669065:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321225.669113:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398c380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983270144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321225.669134:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1456 00000100:00100000:2.0:1767321225.669179:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463a8e3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983270144:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1767321225.669473:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321225.669819:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983270272:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321225.669854:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321225.669932:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270272 00000100:00100000:3.0:1767321225.669937:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270272 00000100:00100000:3.0:1767321225.669982:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1457 00000100:00100000:3.0:1767321225.669990:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983270272:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321225.670027:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321225.670062:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983270272:12345-0@lo:13:osp-pre-0-1.0 Request processed in 73us (207us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321225.670069:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1457 00000100:00080000:0.0:1767321225.670720:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:0.0:1767321225.670761:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983270272:0@lo:13:osp-pre-0-1.0 00010000:00010000:2.0:1767321226.372813:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321227.399850:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321228.229343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321228.229353:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321228.229437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321228.229441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321228.229445:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321228.229453:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6471 00000100:00080000:2.0:1767321228.229457:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321228.229464:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321228.229487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff934604358a80 x1853163983270528/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1767321228.229495:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983270400:0@lo:400:kworker.0 00000100:00080000:2.0:1767321228.229530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321228.229536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321228.229538:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321228.229542:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6471 00000100:00080000:2.0:1767321228.229545:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321228.229549:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321228.229574:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912300 x1853163983270784/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321228.229583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321228.229586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767321228.229590:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321228.229591:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321228.229597:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6471 00000100:00080000:2.0:1767321228.229601:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1767321228.229605:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321228.229618:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910a80 x1853163983270912/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321228.229643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321228.229648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1767321228.229649:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1767321228.229650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321228.229656:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6471 00000100:00080000:2.0:1767321228.229660:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 5s 00000100:00100000:3.0:1767321228.229660:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983271168:0@lo:400:kworker.0 00000100:00080000:2.0:1767321228.229668:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321228.229682:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983271296/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321228.229724:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983270656:0@lo:400:kworker.0 00000100:00100000:3.0:1767321228.229729:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321228.229759:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321228.229782:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270400 00000100:00100000:2.0:1767321228.229785:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321228.229790:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270400 00000100:00100000:2.0:1767321228.229793:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983271040:0@lo:400:kworker.0 00000100:00100000:3.0:1767321228.229851:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270656 00000100:00100000:3.0:1767321228.229854:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270656 00000100:00100000:3.0:1767321228.229886:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2187 00000100:00100000:3.0:1767321228.229893:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983270400:12345-0@lo:400:kworker.0 00000100:00080000:0.0:1767321228.229921:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:3.0:1767321228.229944:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983270400:12345-0@lo:400:kworker.0 Request processed in 50us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321228.229953:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2187 00000100:00100000:0.0:1767321228.229965:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271168 00000100:00100000:0.0:1767321228.229971:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271168 00000100:00100000:3.0:1767321228.229973:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2188 00000100:00100000:3.0:1767321228.229984:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983270656:12345-0@lo:400:kworker.0 00000100:00100000:3.0:1767321228.230013:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983270656:12345-0@lo:400:kworker.0 Request processed in 32us (254us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321228.230018:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2188 00000100:00100000:0.0:1767321228.230031:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1345 00000100:00100000:0.0:1767321228.230041:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983271168:12345-0@lo:400:kworker.0 00010000:00080000:3.0:1767321228.230056:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460435b100 x1853163983270400/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321244 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321228.230071:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460435b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983270400:0@lo:400:kworker.0 00000100:00100000:0.0:1767321228.230082:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463a8e3100 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983271168:12345-0@lo:400:kworker.0 Request processed in 43us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321228.230091:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1345 00000100:00100000:3.0:1767321228.230099:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983271168:0@lo:400:kworker.0 00000100:00080000:0.0:1767321228.230114:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:1.0:1767321228.230261:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983270528:0@lo:38:kworker.0 00000100:00100000:1.0:1767321228.230319:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321228.230336:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983270784:0@lo:38:kworker.0 00000100:00100000:1.0:1767321228.230354:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321228.230377:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983270912:0@lo:38:kworker.0 00000100:00100000:1.0:1767321228.230398:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321228.230414:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270528 00000100:00100000:3.0:1767321228.230420:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270528 00000100:00100000:1.0:1767321228.230436:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983271296:0@lo:38:kworker.0 00000100:00100000:3.0:1767321228.230439:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270784 00000100:00100000:3.0:1767321228.230443:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270784 00000100:00100000:1.0:1767321228.230463:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321228.230463:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2898 00000100:00100000:3.0:1767321228.230472:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983270528:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321228.230489:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321228.230514:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271296 00000100:00100000:1.0:1767321228.230519:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271296 00000100:00100000:3.0:1767321228.230524:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ce00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983270528:12345-0@lo:38:kworker.0 Request processed in 50us (202us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321228.230531:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2898 00000100:00100000:2.0:1767321228.230592:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:2.0:1767321228.230617:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911f80 x1853163983270656/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321244 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1767321228.230627:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983270656:0@lo:400:kworker.0 00000100:00100000:2.0:1767321228.230647:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983270912 00000100:00100000:2.0:1767321228.230651:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983270912 00000100:00100000:2.0:1767321228.230672:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 749 00000100:00100000:2.0:1767321228.230676:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983270912:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321228.230687:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321228.230724:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983270912:12345-0@lo:38:kworker.0 Request processed in 46us (325us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321228.230731:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 749 00000100:00100000:2.0:1767321228.230749:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271040 00000100:00100000:2.0:1767321228.230752:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271040 00000100:00100000:2.0:1767321228.230779:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1346 00000100:00100000:2.0:1767321228.230785:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983271040:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321228.230831:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a910380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983271040:12345-0@lo:400:kworker.0 Request processed in 46us (239us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321228.230837:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1346 00000100:00080000:2.0:1767321228.230849:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:2.0:1767321228.230868:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983271040:0@lo:400:kworker.0 00000100:00100000:1.0:1767321228.236507:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2899 00000100:00100000:1.0:1767321228.236521:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983270784:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321228.236545:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321228.236632:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398ca80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983270784:12345-0@lo:38:kworker.0 Request processed in 111us (6275us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321228.236639:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2899 00000100:00100000:1.0:1767321228.236661:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2900 00000100:00100000:1.0:1767321228.236665:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983271296:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321228.236676:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321228.236714:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398d180 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:7010:x1853163983271296:12345-0@lo:38:kworker.0 Request processed in 47us (6254us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321228.236720:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2900 00000100:00080000:1.0:1767321228.236771:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321228.236776:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321228.236781:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934604358a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983270528:0@lo:38:kworker.0 00000100:00080000:1.0:1767321228.236792:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321228.236796:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321228.236799:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983270784:0@lo:38:kworker.0 00000100:00080000:1.0:1767321228.236809:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321228.236826:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321228.236829:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983270912:0@lo:38:kworker.0 00000100:00080000:1.0:1767321228.236897:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321228.236902:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321228.236905:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983271296:0@lo:38:kworker.0 00010000:00010000:1.0:1767321228.420492:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321229.196663:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321229.196909:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:0.0:1767321229.200446:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726208 00000100:00100000:0.0:1767321229.200460:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726208 00000100:00100000:0.0:1767321229.201051:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726080 00000100:00100000:0.0:1767321229.201056:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726080 00000100:00100000:1.0:1767321229.201194:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2189 00000100:00100000:1.0:1767321229.201206:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915726208:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321229.201324:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934604358a80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:ee100a64-1c8d-4519-95f7-836c165fda80+6:2509:x1853163915726208:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 122us (4671us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321229.201331:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2189 00000100:00100000:1.0:1767321229.201352:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2901 00000100:00100000:1.0:1767321229.201356:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915726080:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321229.201377:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321229.201399:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460435b100 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:2507:x1853163915726080:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 42us (4491us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321229.201404:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2901 00000100:00080000:3.0:1767321229.201617:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321229.444520:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321230.469651:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321230.790183:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983271424:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321230.790426:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321230.790993:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983271552:0@lo:13:osp-pre-0-1.0 00000100:00100000:0.0:1767321230.791209:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321230.791570:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271424 00000100:00100000:3.0:1767321230.791579:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271424 00000100:00100000:3.0:1767321230.791644:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271552 00000100:00100000:3.0:1767321230.791647:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271552 00000100:00100000:1.0:1767321230.791848:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1458 00000100:00100000:1.0:1767321230.791974:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983271424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1767321230.792013:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321230.792140:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017480 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983271424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 170us (1714us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321230.792251:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1459 00000100:00080000:0.0:1767321230.792541:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:0.0:1767321230.792547:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321230.792591:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93463d968000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983271424:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321230.792980:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983271552:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321230.793016:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321230.793146:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1458 00000100:00100000:3.0:1767321230.793833:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014380 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983271552:12345-0@lo:13:osp-pre-0-1.0 Request processed in 856us (2619us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321230.793841:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1459 00000100:00100000:0.0:1767321230.793947:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983271552:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321231.492563:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321232.522785:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321233.348379:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321233.348390:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1767321233.348453:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321233.348457:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321233.348460:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321233.348469:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6476 00000100:00080000:2.0:1767321233.348473:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321233.348480:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321233.348500:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912d80 x1853163983271808/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321233.348505:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321233.348522:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983271680:0@lo:400:kworker.0 00000100:00080000:2.0:1767321233.348549:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321233.348552:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321233.348554:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321233.348557:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6476 00000100:00080000:2.0:1767321233.348560:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321233.348563:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321233.348572:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a911180 x1853163983272064/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321233.348580:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321233.348582:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321233.348584:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1767321233.348585:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1767321233.348587:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6476 00000100:00080000:2.0:1767321233.348590:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767321233.348594:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:0.0:1767321233.348680:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271680 00000100:00100000:2.0:1767321233.348685:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912a00 x1853163983272192/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321233.348687:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271680 00000100:00080000:2.0:1767321233.348712:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321233.348715:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321233.348716:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321233.348720:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6476 00000100:00080000:2.0:1767321233.348722:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1767321233.348726:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321233.348736:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912680 x1853163983272576/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321233.348740:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190 00000100:00100000:0.0:1767321233.348749:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983271680:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.348796:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0000_UUID+5:7011:x1853163983271680:12345-0@lo:400:kworker.0 Request processed in 48us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321233.348803:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190 00000100:00080000:0.0:1767321233.348825:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321233.348837:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983272448:0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.348863:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321233.348943:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983272320:0@lo:400:kworker.0 00000100:00100000:3.0:1767321233.349096:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983271808:0@lo:38:kworker.0 00000100:00100000:3.0:1767321233.349124:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321233.349154:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983272064:0@lo:38:kworker.0 00000100:00100000:3.0:1767321233.349173:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321233.349187:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983272192:0@lo:38:kworker.0 00000100:00100000:3.0:1767321233.349208:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321233.349225:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983272576:0@lo:38:kworker.0 00000100:00100000:3.0:1767321233.349240:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321233.349262:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272192 00000100:00100000:3.0:1767321233.349265:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272192 00000100:00100000:3.0:1767321233.349305:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 750 00000100:00100000:3.0:1767321233.349310:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9345013f0050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983272192:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321233.349328:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321233.349356:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9345013f0050 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983272192:12345-0@lo:38:kworker.0 Request processed in 45us (147us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321233.349362:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 750 00000100:00100000:3.0:1767321233.349565:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271808 00000100:00100000:3.0:1767321233.349569:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271808 00000100:00100000:3.0:1767321233.349583:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272064 00000100:00100000:3.0:1767321233.349585:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272064 00000100:00100000:3.0:1767321233.349594:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272576 00000100:00100000:3.0:1767321233.349596:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272576 00000100:00100000:3.0:1767321233.349712:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2902 00000100:00100000:3.0:1767321233.349717:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983271808:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321233.349727:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321233.349731:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2903 00000100:00100000:1.0:1767321233.349736:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983272064:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321233.349745:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321233.349751:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463d968000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983271808:12345-0@lo:38:kworker.0 Request processed in 33us (627us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321233.349770:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983272064:12345-0@lo:38:kworker.0 Request processed in 33us (596us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321233.349771:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2902 00000100:00100000:1.0:1767321233.349775:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2903 00000100:00100000:1.0:1767321233.349786:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2904 00000100:00100000:1.0:1767321233.349790:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983272576:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321233.349797:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321233.349819:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d7480 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983272576:12345-0@lo:38:kworker.0 Request processed in 29us (578us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321233.349825:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2904 00000100:00080000:1.0:1767321233.349880:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321233.349884:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321233.349887:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983271808:0@lo:38:kworker.0 00000100:00080000:1.0:1767321233.349894:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321233.349897:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321233.349899:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983272064:0@lo:38:kworker.0 00000100:00080000:1.0:1767321233.349906:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321233.349909:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321233.349911:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983272192:0@lo:38:kworker.0 00000100:00080000:1.0:1767321233.349917:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321233.349920:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321233.349921:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983272576:0@lo:38:kworker.0 00010000:00080000:0.0:1767321233.355340:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911f80 x1853163983271680/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321249 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321233.355360:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0000_UUID:7011:1853163983271680:0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.355469:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272448 00000100:00100000:0.0:1767321233.355476:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272448 00000100:00100000:0.0:1767321233.355536:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347 00000100:00100000:0.0:1767321233.355545:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983272448:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.355639:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016680 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983272448:12345-0@lo:400:kworker.0 Request processed in 94us (6773us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321233.355647:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347 00000100:00080000:0.0:1767321233.355672:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:0.0:1767321233.355694:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983272448:0@lo:400:kworker.0 00000100:00100000:1.0:1767321233.356309:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321233.356358:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1767321233.356369:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983271936:0@lo:400:kworker.0 00000100:00100000:1.0:1767321233.356416:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321233.356466:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983271936 00000100:00100000:1.0:1767321233.356473:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983271936 00000100:00100000:1.0:1767321233.356515:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2191 00000100:00100000:1.0:1767321233.356522:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983271936:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321233.356563:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398f800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983271936:12345-0@lo:400:kworker.0 Request processed in 40us (146us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321233.356570:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2191 00000100:00080000:1.0:1767321233.356593:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321233.356628:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911c00 x1853163983271936/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321249 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321233.356639:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983271936:0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.359144:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272320 00000100:00100000:0.0:1767321233.359158:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272320 00000100:00100000:0.0:1767321233.359251:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1348 00000100:00100000:0.0:1767321233.359264:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983272320:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321233.359399:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398fb80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983272320:12345-0@lo:400:kworker.0 Request processed in 138us (3091us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321233.359406:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1348 00000100:00100000:3.0:1767321233.359446:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983272320:0@lo:400:kworker.0 00000100:00080000:1.0:1767321233.361334:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00010000:00010000:3.0:1767321233.540509:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321234.318112:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321234.318208:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726464 00000100:00100000:1.0:1767321234.318219:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726464 00000100:00100000:1.0:1767321234.318321:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2192 00000100:00100000:1.0:1767321234.318332:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915726464:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321234.318416:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915726464:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 85us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321234.318425:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2192 00000100:00100000:3.0:1767321234.319202:0:7004:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321234.319289:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726336 00000100:00100000:1.0:1767321234.319296:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726336 00000100:00100000:1.0:1767321234.319331:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2905 00000100:00100000:1.0:1767321234.319338:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915726336:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321234.319358:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321234.319407:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:2507:x1853163915726336:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 68us (209us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321234.319414:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2905 00000100:00080000:0.0:1767321234.319623:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:1.0:1767321234.567451:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1767321235.588908:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321235.912698:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321235.912727:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983272704:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321235.912797:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321235.912931:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272704 00000100:00100000:2.0:1767321235.912939:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272704 00000100:00100000:2.0:1767321235.913002:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1460 00000100:00100000:2.0:1767321235.913017:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983272704:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1767321235.913051:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:2.0:1767321235.913105:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983272704:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (307us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321235.913113:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1460 00000100:00100000:2.0:1767321235.913162:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983272704:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1767321235.913235:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321235.913361:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983272832:0@lo:13:osp-pre-0-1.0 00000100:00100000:3.0:1767321235.913396:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321235.913502:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272832 00000100:00100000:1.0:1767321235.913513:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272832 00000100:00100000:1.0:1767321235.913575:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1461 00000100:00100000:1.0:1767321235.913584:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983272832:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321235.913604:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321235.913646:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5500 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983272832:12345-0@lo:13:osp-pre-0-1.0 Request processed in 64us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321235.913652:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1461 00000100:00080000:0.0:1767321235.913702:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:3.0:1767321235.913753:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983272832:0@lo:13:osp-pre-0-1.0 00010000:00010000:3.0:1767321236.612697:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321237.636443:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1767321238.468525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321238.468536:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1767321238.468641:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983272960:0@lo:400:kworker.0 00000100:00100000:0.0:1767321238.468721:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321238.468800:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983272960 00000100:00080000:3.0:1767321238.468805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1767321238.468808:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983272960 00000100:00080000:3.0:1767321238.468812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321238.468817:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321238.468824:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6481 00000100:00080000:3.0:1767321238.468829:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767321238.468834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321238.468858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4000 x1853163983273088/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321238.468883:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193 00000100:00100000:2.0:1767321238.468904:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983272960:12345-0@lo:400:kworker.0 00000100:00080000:3.0:1767321238.468905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321238.468913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321238.468916:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321238.468921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6481 00000100:00080000:3.0:1767321238.468924:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1767321238.468928:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321238.468954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5880 x1853163983273344/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321238.469010:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983273088:0@lo:38:kworker.0 00000100:00100000:1.0:1767321238.469042:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321238.469067:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983273344:0@lo:38:kworker.0 00000100:00100000:1.0:1767321238.469088:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321238.469116:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273088 00000100:00100000:1.0:1767321238.469120:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273088 00000100:00100000:1.0:1767321238.469151:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273344 00000100:00100000:1.0:1767321238.469154:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273344 00000100:00100000:1.0:1767321238.469168:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2906 00000100:00100000:1.0:1767321238.469174:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983273088:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321238.469207:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321238.469242:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463398df80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983273088:12345-0@lo:38:kworker.0 Request processed in 67us (198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321238.469250:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2906 00000100:00100000:1.0:1767321238.469266:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2907 00000100:00100000:1.0:1767321238.469290:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983273344:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321238.469300:0:73811:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321238.469325:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c624000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:0+-99:7010:x1853163983273344:12345-0@lo:38:kworker.0 Request processed in 53us (235us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321238.469331:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2907 00000100:00080000:1.0:1767321238.469372:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321238.469378:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321238.469382:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983273088:0@lo:38:kworker.0 00000100:00080000:1.0:1767321238.469390:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321238.469393:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321238.469395:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983273344:0@lo:38:kworker.0 00000100:00080000:1.0:1767321238.469428:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321238.469464:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983273216:0@lo:400:kworker.0 00000100:00100000:1.0:1767321238.469489:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321238.469529:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273216 00000100:00100000:1.0:1767321238.469533:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273216 00000100:00100000:1.0:1767321238.469559:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2194 00000100:00100000:1.0:1767321238.469565:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983273216:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321238.469598:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983273216:12345-0@lo:400:kworker.0 Request processed in 33us (108us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321238.469605:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2194 00000100:00080000:1.0:1767321238.469619:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00080000:1.0:1767321238.469635:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4380 x1853163983273216/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321254 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321238.469645:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983273216:0@lo:400:kworker.0 00000100:00080000:3.0:1767321238.470593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321238.470600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321238.470602:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321238.470607:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6481 00000100:00080000:3.0:1767321238.470611:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767321238.470616:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321238.470640:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163983273472/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321238.470972:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983273472:0@lo:38:kworker.0 00000100:00100000:1.0:1767321238.471022:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321238.471101:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273472 00000100:00100000:1.0:1767321238.471107:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273472 00000100:00100000:1.0:1767321238.471138:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 751 00000100:00100000:1.0:1767321238.471143:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983273472:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321238.471159:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321238.471194:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626a00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983273472:12345-0@lo:38:kworker.0 Request processed in 50us (172us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321238.471200:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 751 00000100:00080000:1.0:1767321238.471233:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321238.471238:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321238.471242:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983273472:0@lo:38:kworker.0 00000100:00100000:2.0:1767321238.471286:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983272960:12345-0@lo:400:kworker.0 Request processed in 2368us (2547us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321238.471297:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193 00000100:00100000:1.0:1767321238.471304:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983273600:0@lo:400:kworker.0 00010000:00080000:0.0:1767321238.471326:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d6d80 x1853163983272960/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321254 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321238.471334:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321238.471338:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983272960:0@lo:400:kworker.0 00000100:00100000:2.0:1767321238.471344:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983273728:0@lo:400:kworker.0 00000100:00100000:0.0:1767321238.471363:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273600 00000100:00100000:0.0:1767321238.471368:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273600 00000100:00100000:2.0:1767321238.471376:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321238.471421:0:72078:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273728 00000100:00100000:0.0:1767321238.471425:0:72078:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273728 00000100:00100000:2.0:1767321238.471431:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1349 00000100:00100000:2.0:1767321238.471440:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983273600:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321238.471451:0:72078:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1350 00000100:00100000:0.0:1767321238.471461:0:72078:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983273728:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321238.471480:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7012:x1853163983273600:12345-0@lo:400:kworker.0 Request processed in 41us (145us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321238.471487:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1349 00000100:00100000:0.0:1767321238.471523:0:72078:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911c00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983273728:12345-0@lo:400:kworker.0 Request processed in 62us (144us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321238.471530:0:72078:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1350 00000100:00100000:0.0:1767321238.471638:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983273728:0@lo:400:kworker.0 00000100:00080000:1.0:1767321238.471745:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00080000:1.0:1767321238.471749:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321238.471770:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:7012:1853163983273600:0@lo:400:kworker.0 00000100:00080000:3.0:1767321238.471823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321238.471828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321238.471830:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321238.471835:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6481 00000100:00080000:3.0:1767321238.471838:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1767321238.471842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321238.471860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7800 x1853163983273856/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321238.472377:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983273856:0@lo:38:kworker.0 00000100:00100000:1.0:1767321238.472410:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321238.472490:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273856 00000100:00100000:2.0:1767321238.472496:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273856 00000100:00100000:2.0:1767321238.472536:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2908 00000100:00100000:2.0:1767321238.472543:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983273856:12345-0@lo:38:kworker.0 00010000:00020000:2.0:1767321238.472556:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1767321238.472600:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c625c00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983273856:12345-0@lo:38:kworker.0 Request processed in 56us (188us total) trans 0 rc -19/-19 00000100:00100000:2.0:1767321238.472606:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2908 00000100:00080000:3.0:1767321238.472778:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1767321238.472783:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:3.0:1767321238.472786:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983273856:0@lo:38:kworker.0 00010000:00010000:2.0:1767321238.661855:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1767321239.440422:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:2.0:1767321239.440460:0:7005:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321239.440556:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726720 00000400:00000100:2.0:1767321239.440559:0:7005:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-192.168.204.7@tcp portal 26 match 1853163915726848 offset 0 length 520: 4 00000100:00100000:1.0:1767321239.440567:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726720 00000100:00100000:3.0:1767321239.440653:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726592 00000100:00100000:3.0:1767321239.440869:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726592 00000100:00080000:0.0:1767321239.441296:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321239.441327:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2195 00000100:00100000:1.0:1767321239.441339:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915726720:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:3.0:1767321239.441837:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2909 00000100:00100000:3.0:1767321239.441843:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915726592:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:3.0:1767321239.441866:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321239.442244:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a913b80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:ee100a64-1c8d-4519-95f7-836c165fda80+6:2510:x1853163915726720:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 904us (1831us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321239.442246:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a911f80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:2507:x1853163915726592:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 396us (1780us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321239.442256:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2195 00000100:00100000:3.0:1767321239.442264:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2909 00010000:00010000:3.0:1767321239.685182:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321240.713490:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1767321241.029318:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983273984:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1767321241.029384:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321241.029420:0:76598:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983273984 00000100:00100000:2.0:1767321241.029429:0:76598:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983273984 00000100:00100000:2.0:1767321241.029514:0:76598:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1462 00000100:00100000:2.0:1767321241.029524:0:76598:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983273984:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321241.029535:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983274112:0@lo:13:osp-pre-0-1.0 00002000:00000024:2.0:1767321241.029556:0:76598:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321241.029570:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321241.029618:0:76598:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633016300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983273984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (232us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321241.029625:0:76598:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1462 00000100:00100000:1.0:1767321241.029630:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274112 00000100:00100000:1.0:1767321241.029635:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274112 00000100:00100000:1.0:1767321241.029668:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1463 00000100:00100000:2.0:1767321241.029674:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983273984:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321241.029678:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983274112:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:1.0:1767321241.029698:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:1.0:1767321241.029733:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93463c626d80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983274112:12345-0@lo:13:osp-pre-0-1.0 Request processed in 58us (163us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321241.029741:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1463 00000100:00100000:2.0:1767321241.029753:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983274112:0@lo:13:osp-pre-0-1.0 00000100:00080000:3.0:1767321241.030009:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00080000:3.0:1767321241.030014:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00010000:00010000:2.0:1767321241.732604:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321242.564398:0:7010:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1767321217/real 1767321217] req@ffff934633017800 x1853163983267712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321242 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321242.564442:0:7010:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff934633017800 x1853163983267712/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 1 dl 1767321242 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1767321242.564466:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.564492:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 0@lo failed (-110) 00000100:00100000:1.0:1767321242.564497:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff934633017800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983267712:0@lo:250:kworker.0 00000100:00080000:3.0:1767321242.564591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.564598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1767321242.564601:0:37:0:(import.c:65:import_set_state_nolock()) ffff9346040e6000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.564606:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.107@tcp: connect to NID 0@lo last attempt 6466 00000100:00080000:3.0:1767321242.564610:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.107@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.564617:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.107@tcp: import ffff9346040e6000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321242.564653:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d6a00 x1853163983274240/t0(0) o250->MGC192.168.204.107@tcp@0@lo:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321242.564750:0:7012:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983274368:0@lo:400:kworker.0 00000100:00100000:0.0:1767321242.564816:0:7012:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321242.564966:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3c2858f3-33b3-41be-8fcd-d8867d4e7e77:7010:1853163983274240:0@lo:250:kworker.0 00000400:00000100:1.0:1767321242.564989:0:7010:0:(lib-move.c:4502:lnet_parse_put()) Dropping PUT from 12345-0@lo portal 26 match 1853163983274240 offset 0 length 520: 4 00000100:00080000:3.0:1767321242.565051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.565055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.565057:0:37:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.565061:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6487 00000100:00080000:3.0:1767321242.565064:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321242.565068:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321242.565086:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7100 x1853163983274496/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321242.565117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.565121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.565123:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.565127:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6487 00000100:00080000:3.0:1767321242.565129:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1767321242.565133:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321242.565145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d7800 x1853163983274752/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1767321242.565153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.565156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.565157:0:37:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.565160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6487 00000100:00080000:3.0:1767321242.565162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.565166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321242.565176:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d5180 x1853163983274880/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1767321242.565177:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1767321242.565187:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983274624:0@lo:400:kworker.0 00000100:00100000:2.0:1767321242.565216:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321242.565227:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983275136:0@lo:400:kworker.0 00000100:00100000:2.0:1767321242.565238:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983275008:0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.565256:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1767321242.565267:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274368 00000100:00100000:2.0:1767321242.565298:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274368 00000100:00100000:1.0:1767321242.565316:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275136 00000100:00100000:1.0:1767321242.565320:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275136 00000100:00100000:2.0:1767321242.565345:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274624 00000100:00100000:2.0:1767321242.565347:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274624 00000100:00100000:1.0:1767321242.565348:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1351 00000100:00100000:1.0:1767321242.565355:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983275136:12345-0@lo:400:kworker.0 00000100:00100000:2.0:1767321242.565365:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196 00000100:00100000:2.0:1767321242.565369:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983274368:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.565400:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f0e00 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983275136:12345-0@lo:400:kworker.0 Request processed in 45us (143us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321242.565412:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1351 00000100:00100000:1.0:1767321242.565452:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2197 00000100:00100000:1.0:1767321242.565460:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983274624:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.565489:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff93460a912680 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:7014:x1853163983274624:12345-0@lo:400:kworker.0 Request processed in 31us (272us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321242.565495:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2197 00000100:00100000:1.0:1767321242.565530:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983275136:0@lo:400:kworker.0 00000100:00080000:1.0:1767321242.565570:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:1.0:1767321242.565590:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983274496:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.565614:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321242.565629:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983274752:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.565644:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321242.565655:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983274880:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.565695:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321242.565722:0:77064:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274880 00000100:00100000:1.0:1767321242.565726:0:77064:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274880 00000100:00100000:1.0:1767321242.565744:0:77064:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 752 00000100:00100000:1.0:1767321242.565749:0:77064:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983274880:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.565766:0:77064:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.565798:0:77064:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1180 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983274880:12345-0@lo:38:kworker.0 Request processed in 48us (102us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.565804:0:77064:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 752 00000100:00100000:1.0:1767321242.565821:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274496 00000100:00100000:1.0:1767321242.565825:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274496 00000100:00100000:1.0:1767321242.565840:0:73813:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983274752 00000100:00100000:1.0:1767321242.565842:0:73813:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983274752 00000100:00100000:1.0:1767321242.565854:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2910 00000100:00100000:1.0:1767321242.565858:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983274496:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.565867:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.565891:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2d80 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983274496:12345-0@lo:38:kworker.0 Request processed in 32us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.565896:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2910 00000100:00100000:1.0:1767321242.565906:0:73813:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2911 00000100:00100000:1.0:1767321242.565910:0:73813:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983274752:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321242.565918:0:73813:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321242.565936:0:73813:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2300 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:0+-99:7010:x1853163983274752:12345-0@lo:38:kworker.0 Request processed in 25us (291us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321242.565942:0:73813:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2911 00000100:00080000:1.0:1767321242.565967:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.565971:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.565974:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983274496:0@lo:38:kworker.0 00000100:00080000:1.0:1767321242.565982:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.565985:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.565988:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983274752:0@lo:38:kworker.0 00000100:00080000:1.0:1767321242.565995:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.565997:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.566000:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983274880:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.566031:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00080000:1.0:1767321242.566054:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4e00 x1853163983274624/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321258 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1767321242.566062:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:7014:1853163983274624:0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.566081:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275008 00000100:00100000:1.0:1767321242.566084:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275008 00000100:00100000:1.0:1767321242.566103:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352 00000100:00100000:1.0:1767321242.566109:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983275008:12345-0@lo:400:kworker.0 00000100:00100000:1.0:1767321242.566141:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f3b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983275008:12345-0@lo:400:kworker.0 Request processed in 31us (108us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321242.566146:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352 00000100:00100000:1.0:1767321242.566171:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983275008:0@lo:400:kworker.0 00000100:00080000:3.0:1767321242.570536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1767321242.570547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1767321242.570551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1767321242.570559:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6487 00000100:00080000:3.0:1767321242.570563:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1767321242.570571:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:3.0:1767321242.570611:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9346059d4700 x1853163983275264/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1767321242.570745:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983275264:0@lo:38:kworker.0 00000100:00100000:1.0:1767321242.570819:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321242.572103:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275264 00000100:00100000:3.0:1767321242.572119:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275264 00000100:00100000:3.0:1767321242.572177:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2912 00000100:00100000:3.0:1767321242.572185:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983275264:12345-0@lo:38:kworker.0 00010000:00020000:3.0:1767321242.572208:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1767321242.572303:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f2a00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983275264:12345-0@lo:38:kworker.0 Request processed in 117us (1482us total) trans 0 rc -19/-19 00000100:00100000:3.0:1767321242.572310:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2912 00000100:00080000:1.0:1767321242.572417:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1767321242.572422:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:1.0:1767321242.572426:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983275264:0@lo:38:kworker.0 00000100:00100000:2.0:1767321242.579577:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633014e00 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7012:x1853163983274368:12345-0@lo:400:kworker.0 Request processed in 14194us (14749us total) trans 0 rc 0/0 00000100:00100000:2.0:1767321242.579594:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196 00010000:00080000:0.0:1767321242.583805:0:7012:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff9346059d4a80 x1853163983274368/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321258 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321242.583825:0:7012:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0000_UUID:7012:1853163983274368:0@lo:400:kworker.0 00010000:00010000:3.0:1767321242.761527:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1767321243.781584:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1767321244.556302:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:3.0:1767321244.556372:0:7006:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.7@tcp (source: 12345-192.168.204.7@tcp) 00000100:00100000:1.0:1767321244.559503:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915727104 00000100:00100000:1.0:1767321244.559518:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915727104 00000100:00100000:1.0:1767321244.559623:0:76600:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163915726976 00000100:00100000:1.0:1767321244.559628:0:76600:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163915726976 00000100:00100000:1.0:1767321244.559656:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2198 00000100:00100000:1.0:1767321244.559666:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915727104:12345-192.168.204.7@tcp:400:kworker.0 00000100:00100000:1.0:1767321244.559797:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:ee100a64-1c8d-4519-95f7-836c165fda80+6:2511:x1853163915727104:12345-192.168.204.7@tcp:400:kworker.0 Request processed in 132us (3538us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321244.559805:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2198 00000100:00100000:1.0:1767321244.559840:0:76600:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.7@tcp, seq: 2913 00000100:00100000:1.0:1767321244.559845:0:76600:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915726976:12345-192.168.204.7@tcp:38:kworker.0 00010000:00020000:1.0:1767321244.559865:0:76600:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 192.168.204.7@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321244.559892:0:76600:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d5880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:0+-99:2507:x1853163915726976:12345-192.168.204.7@tcp:38:kworker.0 Request processed in 44us (3518us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321244.559897:0:76600:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.7@tcp, seq: 2913 00000100:00080000:3.0:1767321244.560688:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00010000:00010000:2.0:1767321244.805329:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321245.833652:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1767321246.148500:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1767321246.148707:0:7014:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1767321246.148726:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983275520:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1767321246.148838:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321246.148902:0:70998:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275520 00000100:00100000:3.0:1767321246.148909:0:70998:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275520 00000100:00100000:3.0:1767321246.148968:0:70998:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1464 00000100:00100000:3.0:1767321246.148978:0:70998:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983275520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1767321246.149007:0:70998:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321246.149049:0:70998:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d6300 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7014:x1853163983275520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (211us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321246.149057:0:70998:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1464 00000100:00080000:3.0:1767321246.149081:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0001 00000100:00100000:3.0:1767321246.149114:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9346059d4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:7014:1853163983275520:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1767321246.151791:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983275392:0@lo:13:osp-pre-0-1.0 00000100:00100000:1.0:1767321246.151863:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1767321246.152045:0:70999:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275392 00000100:00100000:3.0:1767321246.152054:0:70999:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275392 00000100:00100000:3.0:1767321246.152127:0:70999:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1465 00000100:00100000:3.0:1767321246.152138:0:70999:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983275392:12345-0@lo:13:osp-pre-0-1.0 00002000:00000024:3.0:1767321246.152170:0:70999:0:(ofd_obd.c:714:ofd_statfs()) blocks cached 0 granted 278208 pending 0 free 3923513344 avail 3691991040 00000100:00100000:3.0:1767321246.152215:0:70999:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346330f1f80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983275392:12345-0@lo:13:osp-pre-0-1.0 Request processed in 80us (351us total) trans 0 rc 0/0 00000100:00100000:3.0:1767321246.152221:0:70999:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1465 00000100:00080000:1.0:1767321246.152362:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-OST0000 00000100:00100000:1.0:1767321246.153575:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983275392:0@lo:13:osp-pre-0-1.0 00010000:00010000:1.0:1767321246.853647:0:32649:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1767321247.686105:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321247.686116:0:65947:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3c2858f3-33b3-41be-8fcd-d8867d4e7e77->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1767321247.686228:0:7014:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983275648:0@lo:400:kworker.0 00000100:00100000:3.0:1767321247.686327:0:7014:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321247.686435:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275648 00000100:00100000:1.0:1767321247.686443:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275648 00000100:00100000:1.0:1767321247.686515:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2199 00000100:00100000:1.0:1767321247.686526:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983275648:12345-0@lo:400:kworker.0 00000100:00080000:2.0:1767321247.686607:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0000_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321247.686612:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321247.686617:0:65947:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321247.686622:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0000: connect to NID 0@lo last attempt 6491 00000100:00080000:2.0:1767321247.686627:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767321247.686632:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0000: import ffff934607c11000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321247.686685:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a912d80 x1853163983275776/t0(0) o38->lustre-MDT0000-lwp-OST0000@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321247.686718:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-OST0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321247.686722:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321247.686724:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321247.686727:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-OST0001: connect to NID 0@lo last attempt 6491 00000100:00080000:2.0:1767321247.686730:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-OST0001: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1767321247.686733:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-OST0001: import ffff9345011c0000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321247.686743:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910700 x1853163983276032/t0(0) o38->lustre-MDT0000-lwp-OST0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1767321247.686751:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321247.686754:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321247.686755:0:65947:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321247.686758:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-osp-MDT0001: connect to NID 0@lo last attempt 6491 00000100:00080000:2.0:1767321247.686761:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-osp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321247.686764:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-osp-MDT0001: import ffff93460ae77800 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321247.686773:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a913100 x1853163983276160/t0(0) o38->lustre-MDT0000-osp-MDT0001@0@lo:24/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1767321247.686836:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321247.686874:0:7013:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983276416:0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.686906:0:7013:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.686941:0:72074:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983276416 00000100:00100000:0.0:1767321247.686946:0:72074:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983276416 00000100:00100000:0.0:1767321247.686995:0:72074:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1353 00000100:00100000:0.0:1767321247.687002:0:72074:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983276416:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.687040:0:72074:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017b80 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:7013:x1853163983276416:12345-0@lo:400:kworker.0 Request processed in 37us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321247.687047:0:72074:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1353 00000100:00100000:0.0:1767321247.687073:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983275776:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.687115:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.687133:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983276032:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.687151:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.687168:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983276160:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.687210:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.687251:0:71014:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983276160 00000100:00100000:0.0:1767321247.687255:0:71014:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983276160 00000100:00100000:0.0:1767321247.687301:0:71014:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 753 00000100:00100000:0.0:1767321247.687307:0:71014:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983276160:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.687325:0:71014:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.687368:0:71014:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c3800 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_out00_00:0+-99:7010:x1853163983276160:12345-0@lo:38:kworker.0 Request processed in 60us (157us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.687375:0:71014:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 753 00000100:00100000:0.0:1767321247.687395:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275776 00000100:00100000:0.0:1767321247.687399:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275776 00000100:00100000:0.0:1767321247.687415:0:73812:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983276032 00000100:00100000:0.0:1767321247.687417:0:73812:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983276032 00000100:00100000:0.0:1767321247.687433:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2914 00000100:00100000:0.0:1767321247.687437:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983275776:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.687446:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.687471:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633015880 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983275776:12345-0@lo:38:kworker.0 Request processed in 33us (355us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.687476:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2914 00000100:00100000:0.0:1767321247.687487:0:73812:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1767321247.687490:0:73812:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983276032:12345-0@lo:38:kworker.0 00010000:00020000:0.0:1767321247.687499:0:73812:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1767321247.687521:0:73812:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff934633017800 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:0+-99:7010:x1853163983276032:12345-0@lo:38:kworker.0 Request processed in 28us (368us total) trans 0 rc -19/-19 00000100:00100000:0.0:1767321247.687526:0:73812:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1767321247.687564:0:7013:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:7013:1853163983276416:0@lo:400:kworker.0 00000100:00080000:0.0:1767321247.687590:0:7010:0:(import.c:65:import_set_state_nolock()) ffff934607c11000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.687594:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.687597:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a912d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0000_UUID:7010:1853163983275776:0@lo:38:kworker.0 00000100:00080000:0.0:1767321247.687605:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345011c0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.687607:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.687610:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-OST0001_UUID:7010:1853163983276032:0@lo:38:kworker.0 00000100:00080000:0.0:1767321247.687617:0:7010:0:(import.c:65:import_set_state_nolock()) ffff93460ae77800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1767321247.687620:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:0.0:1767321247.687623:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a913100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:7010:1853163983276160:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.687661:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983275904:0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.687687:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.687705:0:7011:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1767321247.687710:0:7011:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983276288:0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.687731:0:7011:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1767321247.687752:0:70993:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983276288 00000100:00100000:0.0:1767321247.687755:0:70993:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983276288 00000100:00100000:0.0:1767321247.687777:0:70993:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1354 00000100:00100000:0.0:1767321247.687783:0:70993:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983276288:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.687815:0:70993:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:7011:x1853163983276288:12345-0@lo:400:kworker.0 Request processed in 32us (83us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321247.687822:0:70993:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1354 00000100:00100000:0.0:1767321247.687837:0:73811:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983275904 00000100:00100000:0.0:1767321247.687840:0:73811:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983275904 00000100:00100000:0.0:1767321247.687865:0:73811:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2200 00000100:00100000:0.0:1767321247.687871:0:73811:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983275904:12345-0@lo:400:kworker.0 00000100:00100000:0.0:1767321247.687898:0:73811:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c1500 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:7011:x1853163983275904:12345-0@lo:400:kworker.0 Request processed in 28us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1767321247.687904:0:73811:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2200 00000100:00080000:0.0:1767321247.687919:0:15482:0:(pinger.c:498:ping_evictor_main()) evicting all exports of obd lustre-MDT0001 00000100:00100000:0.0:1767321247.687935:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:7011:1853163983276288:0@lo:400:kworker.0 00010000:00080000:0.0:1767321247.687945:0:7011:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911880 x1853163983275904/t0(0) o400->lustre-MDT0001-lwp-OST0001@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321263 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1767321247.687954:0:7011:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:7011:1853163983275904:0@lo:400:kworker.0 00000100:00100000:1.0:1767321247.693378:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346059d4000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0000_UUID+5:7014:x1853163983275648:12345-0@lo:400:kworker.0 Request processed in 6813us (7010us total) trans 0 rc 0/0 00000100:00100000:1.0:1767321247.693392:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2199 00000100:00080000:2.0:1767321247.696665:0:65947:0:(pinger.c:229:ptlrpc_pinger_process_import()) lustre-MDT0000-lwp-MDT0001_UUID->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1767321247.696674:0:65947:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1767321247.696679:0:65947:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1767321247.696686:0:65947:0:(import.c:509:import_select_connection()) lustre-MDT0000-lwp-MDT0001: connect to NID 0@lo last attempt 6491 00000100:00080000:2.0:1767321247.696692:0:65947:0:(import.c:588:import_select_connection()) lustre-MDT0000-lwp-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1767321247.696699:0:65947:0:(import.c:623:import_select_connection()) lustre-MDT0000-lwp-MDT0001: import ffff9345087fa000 using connection 192.168.204.107@tcp/0@lo 00000100:00100000:2.0:1767321247.696749:0:65947:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93460a910e00 x1853163983276544/t0(0) o38->lustre-MDT0000-lwp-MDT0001@0@lo:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1767321247.696907:0:7010:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983276544:0@lo:38:kworker.0 00000100:00100000:0.0:1767321247.696966:0:7010:0:(events.c:340:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1767321247.697176:0:74554:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1853163983276544 00000100:00100000:1.0:1767321247.697185:0:74554:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1853163983276544 00000100:00100000:1.0:1767321247.697231:0:74554:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2916 00000100:00100000:1.0:1767321247.697239:0:74554:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983276544:12345-0@lo:38:kworker.0 00010000:00020000:1.0:1767321247.697260:0:74554:0:(ldlm_lib.c:1179:target_handle_connect()) lustre-MDT0000: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1767321247.697351:0:74554:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9346392c0380 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:0+-99:7010:x1853163983276544:12345-0@lo:38:kworker.0 Request processed in 112us (384us total) trans 0 rc -19/-19 00000100:00100000:1.0:1767321247.697357:0:74554:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2916 00000100:00080000:2.0:1767321247.698104:0:7010:0:(import.c:65:import_set_state_nolock()) ffff9345087fa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1767321247.698110:0:7010:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 0@lo failed (-19) 00000100:00100000:2.0:1767321247.698113:0:7010:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a910e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-lwp-MDT0001_UUID:7010:1853163983276544:0@lo:38:kworker.0 00010000:00080000:3.0:1767321247.699471:0:7014:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61550) req@ffff93460a911180 x1853163983275648/t0(0) o400->lustre-MDT0001-lwp-OST0000@0@lo:12/10 lens 224/224 e 0 to 0 dl 1767321263 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1767321247.699487:0:7014:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff93460a911180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0000_UUID:7014:1853163983275648:0@lo:400:kworker.0 00010000:00010000:2.0:1767321247.876950:0:15455:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1767321248.902624:0:15456:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 92750 lines, 92750 kept, 0 dropped, 0 bad.